Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
+1 -2
View File
@@ -400,8 +400,7 @@ class AXK1Attention(nn.Module):
class AXK1MLAAttention(nn.Module):
"""
Main reference: DeepseekV2 paper, and FlashInfer Implementation
"""Main reference: DeepseekV2 paper, and FlashInfer Implementation
(https://arxiv.org/abs/2405.04434 and https://github.com/flashinfer-ai/flashinfer/pull/551).
For more info see MLACommonImpl in:
+6 -9
View File
@@ -39,7 +39,6 @@ _GENERATE_SUFFIXES = [
def _load_st_projector(model_config: "ModelConfig") -> nn.Module | None:
"""Load Sentence-Transformers Dense projection layers."""
dense_modules = try_get_dense_modules(
model_config.model, revision=model_config.revision
)
@@ -249,8 +248,7 @@ def _create_pooling_model_cls(orig_cls: type[_T]) -> type[_T]:
def as_embedding_model(cls: type[_T]) -> type[_T]:
"""
Subclass an existing vLLM model to support embeddings.
"""Subclass an existing vLLM model to support embeddings.
By default, the embeddings of the whole prompt are extracted from the
normalized hidden state corresponding to the last token.
@@ -258,6 +256,7 @@ def as_embedding_model(cls: type[_T]) -> type[_T]:
Note:
We assume that no extra layers are added to the original model;
please implement your own model if this is not the case.
"""
# Avoid modifying existing embedding models
if is_pooling_model(cls):
@@ -306,8 +305,7 @@ def _resolve_num_labels(hf_config: Any, text_config: Any) -> int:
def as_seq_cls_model(cls: type[_T]) -> type[_T]:
"""
Subclass an existing vLLM model to support classify and score tasks.
"""Subclass an existing vLLM model to support classify and score tasks.
By default, the class probabilities are extracted from the softmaxed
hidden state corresponding to the last token.
@@ -316,6 +314,7 @@ def as_seq_cls_model(cls: type[_T]) -> type[_T]:
We assume that the classification head is a single linear layer
stored as the attribute `score` of the top-level model;
please implement your own model if this is not the case.
"""
# Avoid modifying existing classification models
if is_pooling_model(cls):
@@ -446,8 +445,7 @@ class SequenceClassificationConfig(VerifyAndUpdateConfig):
def _get_language_model_for_seq_cls(model: nn.Module) -> nn.Module:
"""
Get the language model component for sequence classification conversion.
"""Get the language model component for sequence classification conversion.
For VLMs, returns the inner language model. For standard LLMs, returns model itself.
"""
multimodal_model: object = model
@@ -481,8 +479,7 @@ def _get_language_model_for_seq_cls(model: nn.Module) -> nn.Module:
@contextmanager
def _disable_seq_cls_loading_on_inner_model(language_model, is_vlm: bool):
"""
Context manager to temporarily disable sequence classification loading
"""Context manager to temporarily disable sequence classification loading
on inner VLM models to prevent recursive seq_cls_model_loader calls.
"""
if not is_vlm:
+10 -5
View File
@@ -47,7 +47,8 @@ from .utils import (
class ArceeMLP(nn.Module):
"""Feed-forward layer for Arcee using ReLU^2 activation
(no gating as in LLaMA)."""
(no gating as in LLaMA).
"""
def __init__(
self,
@@ -95,7 +96,8 @@ class ArceeMLP(nn.Module):
class ArceeDecoderLayer(nn.Module):
"""Transformer decoder block for Arcee, with self-attention and
ReLU^2 MLP."""
ReLU^2 MLP.
"""
def __init__(
self,
@@ -175,7 +177,8 @@ class ArceeDecoderLayer(nn.Module):
@support_torch_compile
class ArceeModel(nn.Module, EagleModelMixin):
"""The transformer model backbone for Arcee (embedding layer + stacked
decoder blocks + final norm)."""
decoder blocks + final norm).
"""
def __init__(
self,
@@ -277,7 +280,8 @@ class ArceeForCausalLM(
nn.Module, SupportsLoRA, SupportsPP, SupportsEagle, SupportsEagle3
):
"""Arcee Model for causal language modeling, integrated with vLLM
runtime."""
runtime.
"""
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
@@ -358,7 +362,8 @@ class ArceeForCausalLM(
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load weights into the model (delegates to inner model and handles
tied embeddings)."""
tied embeddings).
"""
loader = AutoWeightsLoader(self)
# AutoWeightLoader handles weight name remapping, including fusing
# separate q_proj, k_proj, v_proj into qkv_proj
+14 -20
View File
@@ -41,13 +41,12 @@ from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
class AriaImagePixelInputs(TensorSchema):
"""
Dimensions:
- b: Batch size
- n: Number of images
- c: Number of channels
- h: Height of each image
- w: Width of each image
"""Dimensions:
- b: Batch size
- n: Number of images
- c: Number of channels
- h: Height of each image
- w: Width of each image
"""
type: Literal["pixel_values"]
@@ -119,8 +118,7 @@ class AriaProjectorMLP(nn.Module):
class AriaProjector(nn.Module):
"""
A projection module with one cross attention layer and one FFN layer, which
"""A projection module with one cross attention layer and one FFN layer, which
projects ViT's outputs into MoE's inputs.
Args:
@@ -129,6 +127,7 @@ class AriaProjector(nn.Module):
Outputs:
A tensor with the shape of (batch_size, query_number, output_dim)
"""
def __init__(self, config: AriaConfig, prefix: str = "") -> None:
@@ -187,8 +186,7 @@ class AriaProjector(nn.Module):
class AriaTextMoELayer(nn.Module):
"""
Mixture of Experts (MoE) Layer for the AriaMoE model.
"""Mixture of Experts (MoE) Layer for the AriaMoE model.
This layer implements the MoE mechanism, which routes input tokens to
different experts based on a routing algorithm, processes them through the
@@ -227,8 +225,7 @@ class AriaTextMoELayer(nn.Module):
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""
Forward pass of the MoE Layer.
"""Forward pass of the MoE Layer.
Args:
hidden_states: Input tensor of shape
@@ -236,16 +233,15 @@ class AriaTextMoELayer(nn.Module):
Returns:
torch.Tensor: Output tensor after passing through the MoE layer.
"""
"""
router_output = torch.nn.functional.linear(hidden_states, self.router_weight)
return self.experts(hidden_states, router_output)
class AriaTextDecoderLayer(LlamaDecoderLayer):
"""
Custom Decoder Layer for the AriaMoE model which modifies the standard
"""Custom Decoder Layer for the AriaMoE model which modifies the standard
`LlamaDecoderLayer` by replacing the traditional MLP with a Mixture of
Experts (MoE) Layer.
"""
@@ -262,8 +258,7 @@ class AriaTextDecoderLayer(LlamaDecoderLayer):
class AriaTextModel(LlamaModel, SupportsQuant):
"""
Custom LlamaModel for the AriaMoE model which modifies the standard
"""Custom LlamaModel for the AriaMoE model which modifies the standard
LlamaModel by replacing the `LlamaDecoderLayer` with `MoEDecoderLayer`.
"""
@@ -377,8 +372,7 @@ class AriaMultiModalProcessor(BaseMultiModalProcessor[AriaProcessingInfo]):
dummy_inputs=AriaDummyInputsBuilder,
)
class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
"""
Aria model for conditional generation tasks.
"""Aria model for conditional generation tasks.
This model combines a vision tower, a multi-modal projector, and a language
model to perform tasks that involve both image and text inputs.
+11 -16
View File
@@ -71,11 +71,10 @@ from .utils import (
class AudioFlamingo3FeatureInputs(TensorSchema):
"""
Dimensions:
- num_chunks: Number of audio chunks (flattened)
- nmb: Number of mel bins
- num_audios: Number of original audio files
"""Dimensions:
- num_chunks: Number of audio chunks (flattened)
- nmb: Number of mel bins
- num_audios: Number of original audio files
"""
type: Literal["audio_features"]
@@ -96,12 +95,11 @@ class AudioFlamingo3FeatureInputs(TensorSchema):
class AudioFlamingo3EmbeddingInputs(TensorSchema):
"""
Dimensions:
- bn: Batch size
- naf: Number of audio features
- hs: Hidden size (must match the hidden size of language model
backbone)
"""Dimensions:
- bn: Batch size
- naf: Number of audio features
- hs: Hidden size (must match the hidden size of language model
backbone)
"""
type: Literal["audio_embeds"] = "audio_embeds"
@@ -154,8 +152,7 @@ class AudioFlamingo3Encoder(Qwen2AudioEncoder):
return hidden_states
def _get_feat_extract_output_lengths(self, input_lengths: torch.Tensor):
"""
Computes the output length of the convolutional layers and the output length
"""Computes the output length of the convolutional layers and the output length
of the audio encoder
"""
input_lengths = (input_lengths - 1) // 2 + 1
@@ -503,9 +500,7 @@ class AudioFlamingo3ForConditionalGeneration(
}
def get_mm_mapping(self) -> MultiModelKeys:
"""
Get the module prefix in multimodal models
"""
"""Get the module prefix in multimodal models."""
return MultiModelKeys.from_string_field(
language_model="language_model.",
connector="multi_modal_projector.",
+11 -10
View File
@@ -58,12 +58,11 @@ logger = init_logger(__name__)
class BagelImagePixelInputs(TensorSchema):
"""
Dimensions:
- bn: Batch size * number of images
- c: Number of channels (3)
- h: Height of each image
- w: Width of each image
"""Dimensions:
- bn: Batch size * number of images
- c: Number of channels (3)
- h: Height of each image
- w: Width of each image
"""
type: Literal["pixel_values"]
@@ -175,12 +174,12 @@ class PositionEmbedding(nn.Module):
return emb
def forward(self, position_ids: torch.Tensor) -> torch.Tensor:
"""
Args:
"""Args:
position_ids: Flattened position IDs, shape (N,) where each ID
corresponds to a position in the flattened grid
Returns:
Position embeddings of shape (N, hidden_size)
"""
# Ensure position_ids are on the same device as pos_embed
position_ids = position_ids.to(self.pos_embed.device)
@@ -322,8 +321,7 @@ class BagelMultiModalProcessor(BaseMultiModalProcessor[BagelProcessingInfo]):
class BagelForConditionalGeneration(
nn.Module, SupportsMultiModal, SupportsLoRA, SupportsPP
):
"""
BAGEL: A unified multimodal model for image understanding and generation.
"""BAGEL: A unified multimodal model for image understanding and generation.
For vLLM, we focus on the image understanding (vision-to-text) capabilities.
The image generation part is not supported in vLLM.
@@ -507,6 +505,9 @@ class BagelForConditionalGeneration(
positions: Flattened (concatenated) position ids corresponding to a batch.
intermediate_tensors: Intermediate tensors from prior forward pass.
inputs_embeds: Optional tensor of input embeddings.
**kwargs: Multimodal inputs for this batch, forwarded to the
multimodal embedding path.
"""
if intermediate_tensors is not None:
inputs_embeds = None
@@ -75,9 +75,7 @@ def is_linear_layer(layer_idx, layer_group_size):
class BailingMoeV25MLAAttention(nn.Module):
"""
MLA Attention for BailingMoeV2.5 full attention layers.
"""
"""MLA Attention for BailingMoeV2.5 full attention layers."""
def __init__(
self,
@@ -594,7 +592,6 @@ class BailingMoeV25Model(nn.Module):
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load checkpoint weights with simplified mapping."""
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
+4 -4
View File
@@ -458,6 +458,7 @@ class BertEmbeddingModel(nn.Module, SupportsQuant):
Attributes:
model: An instance of BertModel used for forward operations.
_pooler: An instance of Pooler used for pooling operations.
"""
is_pooling_model = True
@@ -575,8 +576,7 @@ class BertMLMHead(nn.Module):
class SPLADESparsePooler(Pooler):
"""
SPLADE sparse pooling:
"""SPLADE sparse pooling:
logits = mlm_head(hidden_states)
-> log1p(relu(logits))
-> (max|sum over L)
@@ -664,8 +664,7 @@ class SPLADESparsePooler(Pooler):
@default_pooling_type(seq_pooling_type="CLS")
class BertSpladeSparseEmbeddingModel(BertEmbeddingModel):
"""
BertEmbeddingModel + SPLADE sparse embedding.
"""BertEmbeddingModel + SPLADE sparse embedding.
- Make logits by self.mlm_head
- pooler: SPLADESparsePooler(mlm_head...)
"""
@@ -779,6 +778,7 @@ class BertForSequenceClassification(nn.Module, SupportsCrossEncoding, SupportsQu
Attributes:
model: An instance of BertModel used for forward operations.
_pooler: An instance of Pooler used for pooling operations.
"""
is_pooling_model = True
+6 -6
View File
@@ -1,7 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Minimal implementation of BlipVisionModel intended to be only used
within a vision language model."""
within a vision language model.
"""
from collections.abc import Iterable
@@ -82,7 +83,7 @@ class BlipVisionEmbeddings(nn.Module):
class BlipAttention(nn.Module):
"""Multi-headed attention from 'Attention Is All You Need' paper"""
"""Multi-headed attention from 'Attention Is All You Need' paper."""
def __init__(
self,
@@ -140,8 +141,7 @@ class BlipAttention(nn.Module):
self,
hidden_states: torch.Tensor,
):
"""Input shape: Batch x Time x Channel"""
"""Input shape: Batch x Time x Channel."""
qkv_states, _ = self.qkv(hidden_states)
query_states, key_states, value_states = qkv_states.chunk(3, dim=-1)
out = self.attn(query_states, key_states, value_states)
@@ -220,12 +220,12 @@ class BlipEncoderLayer(nn.Module):
class BlipEncoder(nn.Module):
"""
Transformer encoder consisting of `config.num_hidden_layers` self
"""Transformer encoder consisting of `config.num_hidden_layers` self
attention layers. Each layer is a [`BlipEncoderLayer`].
Args:
config: BlipConfig
"""
def __init__(
+14 -12
View File
@@ -50,12 +50,11 @@ from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
class Blip2ImagePixelInputs(TensorSchema):
"""
Dimensions:
- bn: Batch size * number of images
- c: Number of channels (3)
- h: Height of each image
- w: Width of each image
"""Dimensions:
- bn: Batch size * number of images
- c: Number of channels (3)
- h: Height of each image
- w: Width of each image
"""
type: Literal["pixel_values"]
@@ -63,11 +62,10 @@ class Blip2ImagePixelInputs(TensorSchema):
class Blip2ImageEmbeddingInputs(TensorSchema):
"""
Dimensions:
- bn: Batch size * number of images
- f: Image feature size
- h: Hidden size (must match the hidden size of language model backbone)
"""Dimensions:
- bn: Batch size * number of images
- f: Image feature size
- h: Hidden size (must match the hidden size of language model backbone)
"""
type: Literal["image_embeds"]
@@ -657,11 +655,15 @@ class Blip2ForConditionalGeneration(
Args:
input_ids: Flattened (concatenated) input_ids corresponding to a
batch.
positions: Flattened (concatenated) position ids corresponding to a
batch.
intermediate_tensors: Intermediate tensors from prior forward pass.
inputs_embeds: Optional tensor of input embeddings.
Info:
[`Blip2ImageInputs`][vllm.model_executor.models.blip2.Blip2ImageInputs]
"""
"""
if intermediate_tensors is not None:
inputs_embeds = None
+8 -10
View File
@@ -67,12 +67,11 @@ from .vision import (
class CLIPImagePixelInputs(TensorSchema):
"""
Dimensions:
- bn: Batch size * number of images
- c: Number of channels (3)
- h: Height of each image
- w: Width of each image
"""Dimensions:
- bn: Batch size * number of images
- c: Number of channels (3)
- h: Height of each image
- w: Width of each image
"""
type: Literal["pixel_values"]
@@ -417,8 +416,7 @@ class CLIPAttention(nn.Module):
self,
hidden_states: torch.Tensor,
):
"""Input shape: Batch x Time x Channel"""
"""Input shape: Batch x Time x Channel."""
qkv_states, _ = self.qkv_proj(hidden_states)
query_states, key_states, value_states = qkv_states.chunk(3, dim=-1)
out = self.attn(query_states, key_states, value_states)
@@ -506,12 +504,12 @@ class CLIPEncoderLayer(nn.Module):
class CLIPEncoder(nn.Module):
"""
Transformer encoder consisting of `config.num_hidden_layers` self
"""Transformer encoder consisting of `config.num_hidden_layers` self
attention layers. Each layer is a [`CLIPEncoderLayer`].
Args:
config: CLIPConfig
"""
def __init__(
+2 -1
View File
@@ -96,7 +96,8 @@ class RMSNorm(nn.Module):
def select_norm_impl(config: CohereConfig) -> tuple[type[nn.Module], float]:
"""Returns (norm_class, eps). Uses RMSNorm when config.rms_norm_eps is set,
otherwise falls back to LayerNorm with config.layer_norm_eps."""
otherwise falls back to LayerNorm with config.layer_norm_eps.
"""
rms_eps = getattr(config, "rms_norm_eps", None)
if rms_eps is not None:
return RMSNorm, rms_eps
+12 -11
View File
@@ -61,14 +61,13 @@ from .utils import (
class Cohere2VisionImagePixelInputs(TensorSchema):
"""
Dimensions:
- np: The total number of patches over each image over each prompt in
the batch
- c: Number of channels
- h: Height of each image patch
- w: Width of each image patch
- bn: Batch size * number of images
"""Dimensions:
- np: The total number of patches over each image over each prompt in
the batch
- c: Number of channels
- h: Height of each image patch
- w: Width of each image patch
- bn: Batch size * number of images
"""
type: Literal["pixel_values"]
@@ -134,6 +133,7 @@ class Cohere2VisionMultiModalProjector(nn.Module):
Returns:
Downsampled tensor with increased channel dimension
"""
height = width = int(image_features.shape[1] ** 0.5)
x = image_features.reshape(image_features.shape[0], width, height, -1)
@@ -175,8 +175,7 @@ class Cohere2VisionProcessingInfo(BaseProcessingInfo):
processor: Cohere2VisionProcessor,
mm_kwargs: Mapping[str, object],
) -> int:
"""
Calculate the number of image patches for a given image.
"""Calculate the number of image patches for a given image.
Uses the HF processor to determine the actual number of patches.
"""
image_processor: Cohere2VisionImageProcessorFast = processor.image_processor
@@ -372,16 +371,18 @@ class Cohere2VisionForConditionalGeneration(
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def _process_image_input(
self, image_input: Cohere2VisionImagePixelInputs, **kwargs
self, image_input: Cohere2VisionImagePixelInputs, **kwargs: object
) -> list[torch.Tensor]:
"""Process image pixels through vision tower and projector.
Args:
image_input: Validated image input containing pixel values and
patch counts
**kwargs: Unused; accepted for interface compatibility.
Returns:
List of flattened image embeddings, one per image
"""
pixel_values = image_input["pixel_values"]
num_patches = image_input["num_patches"]
+36 -18
View File
@@ -300,13 +300,13 @@ class CohereASRMLP(nn.Module):
class FixedPositionalEncoding(nn.Module):
"""
Fixed positional encoding (embedding layer) from sine and cosine functions
"""Fixed positional encoding (embedding layer) from sine and cosine functions
of different frequencies according to https://arxiv.org/abs/1706.03762
Args:
hidden_size: size of the embeddings in the model, also known as d_model
max_sequence_length: maximum allowed length of the input sequence
"""
def __init__(self, hidden_size: int, max_sequence_length: int = 512) -> None:
@@ -664,7 +664,8 @@ class ConvSubsampling(nn.Module):
repeat_num: int = 1,
) -> torch.Tensor:
"""Calculates the output length of a Tensor passed
through a convolution or max pooling layer"""
through a convolution or max pooling layer
"""
add_pad: float = all_paddings - kernel_size
one: float = 1.0
for i in range(repeat_num):
@@ -689,10 +690,12 @@ class ConvSubsampling(nn.Module):
class PositionalEncoding(torch.nn.Module):
"""Fixed sinusoidal positional encoding.
Args:
d_model (int): embedding dim
max_len (int): maximum input length
xscale (bool): whether to scale the input by sqrt(d_model)
"""
def __init__(
@@ -724,12 +727,14 @@ class PositionalEncoding(torch.nn.Module):
self, x: torch.Tensor, cache_len: int = 0
) -> tuple[torch.Tensor, torch.Tensor]:
"""Adds positional encoding.
Args:
x (torch.Tensor): Input. Its shape is (batch, time, feature_size)
cache_len (int): the size of the cache which is used to shift positions
Returns:
x+pos_emb (torch.Tensor): Its shape is (batch, time, feature_size)
pos_emb (torch.Tensor): Its shape is (1, time, feature_size)
"""
input_len = x.size(1) + cache_len
if self.xscale:
@@ -762,14 +767,15 @@ class RelPositionalEncoding(PositionalEncoding):
self, x: torch.Tensor, cache_len: int = 0
) -> tuple[torch.Tensor, torch.Tensor]:
"""Compute positional encoding.
Args:
x (torch.Tensor): Input. Its shape is (batch, time, feature_size)
cache_len (int): the size of the cache which is used to shift positions
Returns:
x (torch.Tensor): Its shape is (batch, time, feature_size)
pos_emb (torch.Tensor): Its shape is (1, time, feature_size)
"""
"""
if self.xscale:
x = x * self.xscale
@@ -783,15 +789,13 @@ class RelPositionalEncoding(PositionalEncoding):
class Swish(nn.SiLU):
"""
Swish activation function introduced in 'https://arxiv.org/abs/1710.05941'
"""Swish activation function introduced in 'https://arxiv.org/abs/1710.05941'
Mathematically identical to SiLU. See note in nn.SiLU for references.
"""
class ConformerFeedForward(nn.Module):
"""
feed-forward module of Conformer model.
"""feed-forward module of Conformer model.
use_bias (bool): Apply bias to all Linear and Conv1d
layers to improve activation flow and stabilize
training of huge models.
@@ -819,8 +823,7 @@ class ConformerFeedForward(nn.Module):
class CausalConv1D(nn.Conv1d):
"""
A causal version of nn.Conv1d where each step would
"""A causal version of nn.Conv1d where each step would
have limited access to locations on its right or left.
All arguments are the same as nn.Conv1d except padding.
@@ -893,6 +896,7 @@ class CausalConv1D(nn.Conv1d):
class ConformerConvolution(nn.Module):
"""The convolution module for the Conformer model.
Args:
d_model (int): hidden dimension
kernel_size (int): kernel size for depthwise convolution
@@ -904,6 +908,7 @@ class ConformerConvolution(nn.Module):
use_bias (bool): Use bias in all Linear and Conv1d
layers to improve activation flow and stabilize
training of huge models. Defaults to True
"""
def __init__(
@@ -979,10 +984,12 @@ class ConformerConvolution(nn.Module):
class CohereASRMultiHeadAttention(nn.Module):
"""Multi-Head Attention layer of Transformer.
Args:
n_head (int): number of heads
n_feat (int): size of the features
use_bias (bool): whether to remove bias in linear and conv layers
"""
def __init__(
@@ -1010,6 +1017,7 @@ class CohereASRMultiHeadAttention(nn.Module):
value: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Transforms query, key and value.
Args:
query (torch.Tensor): (batch, time1, size)
key (torch.Tensor): (batch, time2, size)
@@ -1018,6 +1026,7 @@ class CohereASRMultiHeadAttention(nn.Module):
q (torch.Tensor): (batch, head, time1, size)
k (torch.Tensor): (batch, head, time2, size)
v (torch.Tensor): (batch, head, time2, size)
"""
n_batch = query.size(0)
q = self.linear_q(query).view(n_batch, -1, self.h, self.d_k)
@@ -1036,6 +1045,7 @@ class CohereASRMultiHeadAttention(nn.Module):
mask: torch.Tensor | None,
) -> torch.Tensor:
"""Compute attention context vector.
Args:
value (torch.Tensor): (batch, time2, size)
scores(torch.Tensor): (batch, time1, time2)
@@ -1044,6 +1054,7 @@ class CohereASRMultiHeadAttention(nn.Module):
value (torch.Tensor): transformed `value`
(batch, time2, d_model) weighted by the
attention scores
"""
n_batch = value.size(0)
if mask is not None:
@@ -1071,16 +1082,19 @@ class CohereASRMultiHeadAttention(nn.Module):
pos_emb: torch.Tensor | None = None,
) -> torch.Tensor:
"""Compute 'Scaled Dot Product Attention'.
Args:
query (torch.Tensor): (batch, time1, size)
key (torch.Tensor): (batch, time2, size)
value(torch.Tensor): (batch, time2, size)
mask (torch.Tensor): (batch, time1, time2)
pos_emb (torch.Tensor): optional relative position embeddings
returns:
Returns:
output (torch.Tensor): transformed `value`
(batch, time1, d_model) weighted by the
query dot key attention
"""
q, k, v = self.forward_qkv(query, key, value)
@@ -1130,8 +1144,10 @@ class RelPositionMultiHeadAttention(CohereASRMultiHeadAttention):
def rel_shift(self, x: torch.Tensor) -> torch.Tensor:
"""Compute relative positional encoding.
Args:
x (torch.Tensor): (batch, nheads, time, 2*time-1)
"""
b, h, qlen, pos_len = x.size() # (b, h, t1, t2)
# need to add a column of zeros on the left side of
@@ -1151,17 +1167,19 @@ class RelPositionMultiHeadAttention(CohereASRMultiHeadAttention):
pos_emb: torch.Tensor | None = None,
) -> torch.Tensor:
"""Compute 'Scaled Dot Product Attention' with rel. positional encoding.
Args:
query (torch.Tensor): (batch, time1, size)
key (torch.Tensor): (batch, time2, size)
value(torch.Tensor): (batch, time2, size)
mask (torch.Tensor): (batch, time1, time2)
pos_emb (torch.Tensor) : (batch, time1, size)
pos_emb (torch.Tensor): (batch, time1, size)
Returns:
output (torch.Tensor): transformed `value`
(batch, time1, d_model) weighted by the
query dot key attention
"""
q, k, v = self.forward_qkv(query, key, value)
q = q.transpose(1, 2) # (batch, time1, head, d_k)
@@ -1212,6 +1230,7 @@ class ConformerLayer(torch.nn.Module):
Conv1d layers from each ConformerLayer to
improve activation flow and stabilize training
of huge models. Defaults to True.
"""
def __init__(
@@ -1279,14 +1298,14 @@ class ConformerLayer(torch.nn.Module):
pos_emb: torch.Tensor | None = None,
pad_mask: torch.Tensor | None = None,
) -> torch.Tensor:
"""
Args:
"""Args:
x (torch.Tensor): input signals (B, T, d_model)
att_mask (torch.Tensor): attention masks(B, T, T)
pos_emb (torch.Tensor): (L, 1, d_model)
pad_mask (torch.tensor): padding mask
Returns:
x (torch.Tensor): (B, T, d_model)
"""
residual = x
x = self.norm_feed_forward1(x)
@@ -1331,8 +1350,7 @@ class ConformerLayer(torch.nn.Module):
class ConformerEncoder(nn.Module):
"""
The encoder for ASR model of Conformer.
"""The encoder for ASR model of Conformer.
Based on this paper:
'Conformer: Convolution-augmented Transformer for
Speech Recognition' by Anmol Gulati et al.
@@ -1473,12 +1491,12 @@ class ConformerEncoder(nn.Module):
return num_encoder_cross_attn_tokens
def set_max_audio_length(self, max_audio_length: int) -> None:
"""
Sets maximum input length.
"""Sets maximum input length.
Pre-calculates internal seq_range mask.
Args:
max_audio_length (int): New maximum sequence length.
"""
device = next(self.parameters()).device
dtype = next(self.parameters()).dtype
+9 -12
View File
@@ -338,8 +338,7 @@ def pos_embed_interpolate_native(
class Cohere_VLImagePixelInputs(TensorSchema):
"""
Dimensions:
"""Dimensions:
- np: Number of patches
- ni: Number of images
- cps: Number of channels * patch_size * patch_size
@@ -365,8 +364,7 @@ class Cohere_VLImagePixelInputs(TensorSchema):
class Cohere_VLImageEmbeddingInputs(TensorSchema):
"""
Dimensions:
"""Dimensions:
- nf: Number of image features
- hs: Hidden size
- ni: Number of images
@@ -895,6 +893,7 @@ class Cohere_VisionTransformer(nn.Module):
instead of computing from cu_seqlens (needed for CUDA
graph capture to cover worst-case replay scenarios).
device: Device to place tensors on. Defaults to self.device.
"""
if device is None:
device = self.device
@@ -1744,8 +1743,7 @@ class CohereCompassForConditionalGeneration(
image_embeds_split: tuple[torch.Tensor, ...],
image_input: Cohere_VLImageInputs,
) -> tuple[torch.Tensor, ...]:
"""
Append mrope positions for each for images.
"""Append mrope positions for each for images.
This is necessary to recover correct mrope positions
Args:
@@ -1757,6 +1755,7 @@ class CohereCompassForConditionalGeneration(
Tuple of image embeddings for each image item.
Resulting embeddings will have extra 5 channels for
computed mrope positions, consistent with video embeddings.
"""
if self.is_multimodal_pruning_enabled:
merge_size = self.visual.spatial_merge_size
@@ -1837,8 +1836,7 @@ class CohereCompassForConditionalGeneration(
mrope_positions: torch.LongTensor,
num_computed_tokens: int,
) -> tuple[MultiModalEmbeddings, torch.Tensor, int]:
"""
Update part of input mrope positions (starting with
"""Update part of input mrope positions (starting with
num_computed_tokens index). Original mrope_positions are computed
for unpruned sequence and becomes incorrect once pruning occurs,
so once we prune media tokens we should reflect this in the
@@ -1856,6 +1854,7 @@ class CohereCompassForConditionalGeneration(
Returns:
Tuple of (multimodal_embeddings, mrope_positions,
mrope_position_delta).
"""
return self._recompute_mrope_positions(
input_ids=input_ids,
@@ -2046,8 +2045,8 @@ class CohereCompassForConditionalGeneration(
`None` if no images are passed.
- image_grid_thw: Tensor `(n_images, 3)` of image 3D grid in
LLM. `None` if no images are passed.
"""
"""
if intermediate_tensors is not None:
inputs_embeds = None
@@ -2083,9 +2082,7 @@ class CohereCompassForConditionalGeneration(
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def get_mm_mapping(self) -> MultiModelKeys:
"""
Get the module prefix in multimodal models
"""
"""Get the module prefix in multimodal models"""
return MultiModelKeys.from_string_field(
language_model="language_model",
connector=["visual.merger", "visual.deepstack_merger_list"],
+3 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
ColBERT late interaction model for retrieval and reranking.
"""ColBERT late interaction model for retrieval and reranking.
ColBERT uses per-token embeddings and late interaction (MaxSim) scoring
instead of single-vector representations or cross-encoder concatenation.
@@ -75,6 +74,7 @@ class ColBERTMixin(nn.Module, SupportsLateInteraction):
``None``, will be inferred from weights during loading (or
auto-loaded from sentence-transformers Dense layers).
head_dtype: Data type for the projection layer.
"""
self.hidden_size = hidden_size
self.colbert_dim = colbert_dim
@@ -149,6 +149,7 @@ class ColBERTMixin(nn.Module, SupportsLateInteraction):
Returns:
``(remaining_weights, loaded_names)`` — the weights that were
**not** consumed and the set of names that were loaded.
"""
weights_list = list(weights)
other_weights: list[tuple[str, torch.Tensor]] = []
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
ColPali late interaction model for multi-modal retrieval and reranking.
"""ColPali late interaction model for multi-modal retrieval and reranking.
ColPali extends PaliGemma with a ColBERT-style late interaction head,
producing per-token embeddings for both text and image inputs. It uses
+2 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
ColQwen3 late interaction model for multi-modal retrieval and reranking.
"""ColQwen3 late interaction model for multi-modal retrieval and reranking.
ColQwen3 extends Qwen3-VL with a ColBERT-style late interaction head,
producing per-token embeddings for both text and image inputs. It uses
@@ -131,6 +130,7 @@ class ColQwen3Model(Qwen3VLForConditionalGeneration, SupportsLateInteraction):
Attributes:
custom_text_proj: Linear projection from hidden_size to embed_dim
"""
# Mark this as a pooling model so vLLM routes to pooler path
+2 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
ColQwen3.5 late interaction model for multi-modal retrieval and reranking.
"""ColQwen3.5 late interaction model for multi-modal retrieval and reranking.
ColQwen3.5 extends Qwen3.5 with a ColBERT-style late interaction head,
producing per-token embeddings for both text and image inputs. It uses
@@ -128,6 +127,7 @@ class ColQwen3_5Model(
Attributes:
custom_text_proj: Linear projection from hidden_size to embed_dim.
This is passed to the pooler as pooler.head.projector.
"""
# Mark this as a pooling model so vLLM routes to pooler path
+1 -2
View File
@@ -114,8 +114,7 @@ class RMSNorm(nn.Module):
def select_norm_impl(config: CohereConfig) -> tuple[type[nn.Module], float]:
"""
Returns the normalization layer class and epsilon value to use.
"""Returns the normalization layer class and epsilon value to use.
If `config.rms_norm_eps` is present, use RMSNorm.
Otherwise default to LayerNorm with `config.layer_norm_eps`.
"""
+4 -4
View File
@@ -445,8 +445,7 @@ class GteNewModelConfig(VerifyAndUpdateConfig):
class HybridAttentionMambaModelConfig(VerifyAndUpdateConfig):
@classmethod
def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
"""
Perform early validation and setup for hybrid attention/mamba models.
"""Perform early validation and setup for hybrid attention/mamba models.
Block size alignment with mamba page sizes is handled later by
Platform.update_block_size_for_backend(), which runs after model
@@ -454,6 +453,7 @@ class HybridAttentionMambaModelConfig(VerifyAndUpdateConfig):
Args:
vllm_config: vLLM Config
"""
# Enable FULL_AND_PIECEWISE by default
MambaModelConfig.verify_and_update_config(vllm_config)
@@ -609,12 +609,12 @@ class LlamaNemotronVLConfig(VerifyAndUpdateConfig):
class MambaModelConfig(VerifyAndUpdateConfig):
@classmethod
def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
"""
Enable FULL_AND_PIECEWISE cuda graph mode by default (required
"""Enable FULL_AND_PIECEWISE cuda graph mode by default (required
to get good performance for mamba layers in V1).
Args:
vllm_config: vLLM Config
"""
model_config = vllm_config.model_config
cache_config = vllm_config.cache_config
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Conformer encoder components for FireRedASR2.
"""Conformer encoder components for FireRedASR2.
The audio encoder architecture is
(Conv2dSubsampling → RelPositionalEncoding → N × RelPosEmbConformerBlock).
@@ -298,9 +297,7 @@ class RelPosEmbConformerBlock(nn.Module):
class ConformerEncoder(nn.Module):
"""
Conformer encoder used by FireRedASR2.
"""
"""Conformer encoder used by FireRedASR2."""
def __init__(
self,
+2 -4
View File
@@ -151,8 +151,7 @@ def patch_merging_by_param(
class Cosmos3EdgePatchMerger(nn.Module):
"""
Projector: LayerNorm -> Linear -> GELU -> Linear
"""Projector: LayerNorm -> Linear -> GELU -> Linear.
Reads config from projector_config (not vision_config).
input_hidden_size * spatial_merge_size² -> merger_intermediate_size
@@ -490,8 +489,7 @@ class Cosmos3EdgeForConditionalGeneration(
SupportsPP,
SupportsMRoPE,
):
"""
Cosmos3 Edge model with a SigLIP2 vision encoder.
"""Cosmos3 Edge model with a SigLIP2 vision encoder.
Architecture:
- self.visual: SigLIP2 encoder + patch merger + projector
+62 -60
View File
@@ -248,23 +248,23 @@ class ImageEncoderViT(nn.Module):
global_attn_indexes: tuple[int, ...] = (),
last_conv_output: int = 1024,
) -> None:
"""
Args:
img_size (int): Input image size.
patch_size (int): Patch size.
in_chans (int): Number of input image channels.
embed_dim (int): Patch embedding dimension.
depth (int): Depth of ViT.
num_heads (int): Number of attention heads in each ViT block.
mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
qkv_bias (bool): If True, add a learnable bias to query, key, value.
norm_layer (nn.Module): Normalization layer.
act_layer (nn.Module): Activation layer.
use_abs_pos (bool): If True, use absolute positional embeddings.
use_rel_pos (bool): If True, add relative positional embeddings to the attention map.
rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
window_size (int): Window size for window attention blocks.
global_attn_indexes (list): Indexes for blocks using global attention.
"""Args:
img_size (int): Input image size.
patch_size (int): Patch size.
in_chans (int): Number of input image channels.
embed_dim (int): Patch embedding dimension.
depth (int): Depth of ViT.
num_heads (int): Number of attention heads in each ViT block.
mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
qkv_bias (bool): If True, add a learnable bias to query, key, value.
norm_layer (nn.Module): Normalization layer.
act_layer (nn.Module): Activation layer.
use_abs_pos (bool): If True, use absolute positional embeddings.
use_rel_pos (bool): If True, add relative positional embeddings to the attention map.
rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
window_size (int): Window size for window attention blocks.
global_attn_indexes (list): Indexes for blocks using global attention.
""" # noqa: E501
super().__init__()
self.img_size = img_size
@@ -363,7 +363,8 @@ class ImageEncoderViT(nn.Module):
class Block(nn.Module):
"""Transformer blocks with support of window attention and residual propagation
blocks"""
blocks
"""
def __init__(
self,
@@ -378,20 +379,20 @@ class Block(nn.Module):
window_size: int = 0,
input_size: tuple[int, int] | None = None,
) -> None:
"""
Args:
dim (int): Number of input channels.
num_heads (int): Number of attention heads in each ViT block.
mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
qkv_bias (bool): If True, add a learnable bias to query, key, value.
norm_layer (nn.Module): Normalization layer.
act_layer (nn.Module): Activation layer.
use_rel_pos (bool): If True, add relative positional embeddings to the attention map.
rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
window_size (int): Window size for window attention blocks. If it equals 0, then
use global attention.
input_size (tuple(int, int) or None): Input resolution for calculating the relative
positional parameter size.
"""Args:
dim (int): Number of input channels.
num_heads (int): Number of attention heads in each ViT block.
mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
qkv_bias (bool): If True, add a learnable bias to query, key, value.
norm_layer (nn.Module): Normalization layer.
act_layer (nn.Module): Activation layer.
use_rel_pos (bool): If True, add relative positional embeddings to the attention map.
rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
window_size (int): Window size for window attention blocks. If it equals 0, then
use global attention.
input_size (tuple(int, int) or None): Input resolution for calculating the relative
positional parameter size.
""" # noqa: E501
super().__init__()
self.norm1 = norm_layer(dim)
@@ -448,16 +449,16 @@ class RelPosAttention(PluggableLayer):
rel_pos_zero_init: bool = True,
input_size: tuple[int, int] | None = None,
) -> None:
"""
Args:
dim (int): Number of input channels.
num_heads (int): Number of attention heads.
qkv_bias (bool): If True, add a learnable bias to query, key, value.
use_triton_attention (bool): If True, fuse relative position bias in
a Triton attention kernel on CUDA-alike platforms.
rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
input_size (tuple(int, int) or None): Input resolution for calculating the relative
positional parameter size.
"""Args:
dim (int): Number of input channels.
num_heads (int): Number of attention heads.
qkv_bias (bool): If True, add a learnable bias to query, key, value.
use_triton_attention (bool): If True, fuse relative position bias in
a Triton attention kernel on CUDA-alike platforms.
rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
input_size (tuple(int, int) or None): Input resolution for calculating the relative
positional parameter size.
""" # noqa: E501
super().__init__()
self.num_heads = num_heads
@@ -540,8 +541,8 @@ class RelPosAttention(PluggableLayer):
def window_partition(
x: torch.Tensor, window_size: int
) -> tuple[torch.Tensor, tuple[int, int]]:
"""
Partition into non-overlapping windows with padding if needed.
"""Partition into non-overlapping windows with padding if needed.
Args:
x (tensor): input tokens with [B, H, W, C].
window_size (int): window size.
@@ -549,6 +550,7 @@ def window_partition(
Returns:
windows: windows after partition with [B * num_windows, window_size, window_size, C].
(Hp, Wp): padded height and width before partition
""" # noqa: E501
B, H, W, C = x.shape
@@ -571,8 +573,8 @@ def window_unpartition(
pad_hw: tuple[int, int],
hw: tuple[int, int],
) -> torch.Tensor:
"""
Window unpartition into original sequences and removing padding.
"""Window unpartition into original sequences and removing padding.
Args:
windows (tensor): input tokens with [B * num_windows, window_size, window_size, C].
window_size (int): window size.
@@ -581,6 +583,7 @@ def window_unpartition(
Returns:
x: unpartitioned sequences with [B, H, W, C].
""" # noqa: E501
Hp, Wp = pad_hw
H, W = hw
@@ -596,9 +599,9 @@ def window_unpartition(
def get_rel_pos(q_size: int, k_size: int, rel_pos: torch.Tensor) -> torch.Tensor:
"""
Get relative positional embeddings according to the relative positions of
"""Get relative positional embeddings according to the relative positions of
query and key sizes.
Args:
q_size (int): size of query q.
k_size (int): size of key k.
@@ -606,6 +609,7 @@ def get_rel_pos(q_size: int, k_size: int, rel_pos: torch.Tensor) -> torch.Tensor
Returns:
Extracted positional embeddings according to relative positions.
"""
max_rel_dist = int(2 * max(q_size, k_size) - 1)
# Interpolate rel pos if needed.
@@ -641,8 +645,7 @@ def add_decomposed_rel_pos(
q_size: tuple[int, int],
k_size: tuple[int, int],
) -> torch.Tensor:
"""
Calculate decomposed Relative Positional Embeddings from :paper:`mvitv2`.
"""Calculate decomposed Relative Positional Embeddings from :paper:`mvitv2`.
https://github.com/facebookresearch/mvit/blob/19786631e330df9f3622e5402b4a419a263a2c80/mvit/models/attention.py
Args:
q (Tensor): query q in the attention layer with shape (B, q_h * q_w, C).
@@ -653,6 +656,7 @@ def add_decomposed_rel_pos(
Returns:
attn (Tensor): attention map with added relative positional embeddings.
""" # noqa: E501
q_h, q_w = q_size
k_h, k_w = k_size
@@ -672,9 +676,7 @@ def add_decomposed_rel_pos(
class PatchEmbed(nn.Module):
"""
Image to Patch Embedding.
"""
"""Image to Patch Embedding."""
def __init__(
self,
@@ -684,13 +686,13 @@ class PatchEmbed(nn.Module):
in_chans: int = 3,
embed_dim: int = 768,
) -> None:
"""
Args:
kernel_size (Tuple): kernel size of the projection layer.
stride (Tuple): stride of the projection layer.
padding (Tuple): padding size of the projection layer.
in_chans (int): Number of input image channels.
embed_dim (int): Patch embedding dimension.
"""Args:
kernel_size (Tuple): kernel size of the projection layer.
stride (Tuple): stride of the projection layer.
padding (Tuple): padding size of the projection layer.
in_chans (int): Number of input image channels.
embed_dim (int): Patch embedding dimension.
"""
super().__init__()
+8 -10
View File
@@ -23,8 +23,7 @@ from vllm.model_executor.custom_op import PluggableLayer
# --8<-- [start:qwen2_decoder]
@PluggableLayer.register("qwen2_decoder")
class CustomQwen2Decoder(PluggableLayer):
"""
Qwen2 visual encoder
"""Qwen2 visual encoder
non-causal attention + causal attention
token_type_ids :0=non-causal, 1=causal
"""
@@ -75,7 +74,7 @@ class CustomQwen2Decoder(PluggableLayer):
del self.model.embed_tokens
def _create_custom_model(self, config):
"""Qwen2Model"""
"""Qwen2Model."""
class CustomQwen2ModelInner(Qwen2Model):
def forward(
@@ -194,11 +193,11 @@ class CustomQwen2Decoder(PluggableLayer):
attention_mask: torch.Tensor = None,
**kwargs,
):
"""
Args:
inputs_embeds: [batch_size, seq_len, hidden_dim]
token_type_ids: [batch_size, seq_len], 0=non-causal, 1=causal
attention_mask: [batch_size, seq_len], optional
"""Args:
inputs_embeds: [batch_size, seq_len, hidden_dim]
token_type_ids: [batch_size, seq_len], 0=non-causal, 1=causal
attention_mask: [batch_size, seq_len], optional
"""
return self.model(
inputs_embeds=inputs_embeds,
@@ -209,8 +208,7 @@ class CustomQwen2Decoder(PluggableLayer):
class Qwen2Decoder2Encoder(nn.Module):
"""
Decoder based on Multilingual BART
"""Decoder based on Multilingual BART
Set the initial weights and configuration with a pretrained multilingual BART model,
and modify the detailed configurations as a Nougat decoder
"""
@@ -36,8 +36,7 @@ from .utils import (
class DeepseekV2Eagle3DecoderLayer(nn.Module):
"""
Eagle3 decoder layer for Deepseek that:
"""Eagle3 decoder layer for Deepseek that:
1. Always uses MLP (not MoE)
2. First layer accepts concatenated embeds + hidden_states
"""
+1 -2
View File
@@ -529,8 +529,7 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts, SupportsPP):
return loaded_params
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
"""
Rewrite the weight name to match the format of the original model.
"""Rewrite the weight name to match the format of the original model.
Add .mtp_block for modules in transformer layer block for spec layer
and rename shared layer weights to be top level.
"""
+13 -19
View File
@@ -79,13 +79,12 @@ _IMAGE_TOKEN = "<image>"
class DeepseekOCRImagePixelInputs(TensorSchema):
"""
Dimensions:
- b: Batch size
- n: Number of images
- p: Number of patches
- base_size: Base size of the processor
- image_size: Image size of the processor
"""Dimensions:
- b: Batch size
- n: Number of images
- p: Number of patches
- base_size: Base size of the processor
- image_size: Image size of the processor
"""
type: Literal["pixel_values"]
@@ -140,7 +139,8 @@ class NoRepeatNGramLogitsProcessor:
class NGramPerReqLogitsProcessor(AdapterLogitsProcessor):
"""Example of overriding the wrapper class `__init__()` in order to utilize
info about the device type"""
info about the device type
"""
@classmethod
def validate_params(cls, params: SamplingParams):
@@ -639,9 +639,7 @@ class DeepseekOCRForCausalLM(
return autoloaded_weights
def get_mm_mapping(self) -> MultiModelKeys:
"""
Get the module prefix in multimodal models
"""
"""Get the module prefix in multimodal models."""
return MultiModelKeys.from_string_field(
language_model="language_model",
connector="projector",
@@ -676,8 +674,7 @@ class DeepseekOCRForCausalLM(
self,
image_spatial_crop: torch.Tensor | None = None,
) -> tuple[int, int, int, int]:
"""
Return (num_input_tokens, num_output_tokens, global_output_token,
"""Return (num_input_tokens, num_output_tokens, global_output_token,
local_output_token) for a single image described by
``image_spatial_crop``.
"""
@@ -854,8 +851,7 @@ class DeepseekOCRForCausalLM(
self,
pixel_values: torch.Tensor,
) -> torch.Tensor:
"""
Encode batched global images with newline tokens inserted.
"""Encode batched global images with newline tokens inserted.
Output shape: ``[B * 272, n_embed]``.
"""
bsz = pixel_values.shape[0]
@@ -880,8 +876,7 @@ class DeepseekOCRForCausalLM(
self,
images_crop: torch.Tensor,
) -> torch.Tensor:
"""
Encode local patches without newline insertion (newlines are added later
"""Encode local patches without newline insertion (newlines are added later
in ``postprocess_encoder_output`` via ``_assemble_patch_grid``).
Output shape: ``[P * 100, n_embed]``.
"""
@@ -951,8 +946,7 @@ class DeepseekOCRForCausalLM(
clone: bool = False,
batch_mm_kwargs: dict[str, Any] | None = None,
) -> None:
"""
Assemble per-image embeddings from global and local encoder outputs.
"""Assemble per-image embeddings from global and local encoder outputs.
``output['global']`` contains global-image features with newlines already
inserted (from CUDA graph replay or eager fallback):
+1 -3
View File
@@ -456,9 +456,7 @@ class DeepseekOCR2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, Support
return autoloaded_weights
def get_mm_mapping(self) -> MultiModelKeys:
"""
Get the module prefix in multimodal models
"""
"""Get the module prefix in multimodal models."""
return MultiModelKeys.from_string_field(
language_model="language_model",
connector="projector",
+3 -6
View File
@@ -850,8 +850,7 @@ class Indexer(nn.Module):
def _try_load_fp8_indexer_wk(
name, tensor, buf, params_dict, loaded_params, pp_missing_layer_names
):
"""
We fuse the WK and weights_proj projections, but in some checkpoints WK is stored
"""We fuse the WK and weights_proj projections, but in some checkpoints WK is stored
in FP8 with a separate weight_scale_inv, while weights_proj is stored in BF16.
Upcasting to BF16 during loading enables the fusion. This function loads the FP8 WK
weights and scale, and when both are available, dequantizes to BF16 and stores into
@@ -911,8 +910,7 @@ def _min_latency_fused_qkv_a_proj_impl(
input_: torch.Tensor,
weight: torch.Tensor,
) -> torch.Tensor:
"""
Dynamically run min-latency gemm if num_tokens <= 16.
"""Dynamically run min-latency gemm if num_tokens <= 16.
This must be wrapped in a custom op because our torch.compile integration
does not support runtime dispatching on num_tokens.
"""
@@ -993,8 +991,7 @@ class DeepSeekV2FusedQkvAProjLinear(MergedColumnParallelLinear):
class DeepseekV2MLAAttention(nn.Module):
"""
Main reference: DeepseekV2 paper, and FlashInfer Implementation
"""Main reference: DeepseekV2 paper, and FlashInfer Implementation
(https://arxiv.org/abs/2405.04434 and https://github.com/flashinfer-ai/flashinfer/pull/551).
For more info see MLACommonImpl in:
+10 -12
View File
@@ -64,13 +64,12 @@ _IMAGE_TOKEN = "<image>"
class DeepseekVL2ImagePixelInputs(TensorSchema):
"""
Dimensions:
- bnp: Batch size * number of images * number of patches
- p: Number of patches
- c: Number of channels (3)
- h: Height of each image
- w: Width of each image
"""Dimensions:
- bnp: Batch size * number of images * number of patches
- p: Number of patches
- c: Number of channels (3)
- h: Height of each image
- w: Width of each image
"""
type: Literal["pixel_values"]
@@ -79,11 +78,10 @@ class DeepseekVL2ImagePixelInputs(TensorSchema):
class DeepseekVL2VImageEmbeddingInputs(TensorSchema):
"""
Dimensions:
- bn: Batch size * number of images
- f: Image feature size
- h: Hidden size (must match language model backbone)
"""Dimensions:
- bn: Batch size * number of images
- f: Image feature size
- h: Hidden size (must match language model backbone)
"""
type: Literal["image_embeds"]
@@ -352,7 +352,6 @@ class DiffusionGemmaForConditionalGeneration(
skip duplicate decoder backbone weights, handle vision tower and
self-conditioning separately.
"""
sc_params = dict(
(n, p)
for n, p in self.named_parameters()
@@ -512,7 +511,8 @@ def _compiled_sample_step(
accept/renoise → convergence, all as vectorized PyTorch ops.
Returns the temperature-scaled logits ``[num_decode, CL, vocab]`` so the
caller can compute logprobs outside the compiled region."""
caller can compute logprobs outside the compiled region.
"""
num_decode = decode_slots.shape[0]
device = decode_slots.device
+10 -14
View File
@@ -66,12 +66,11 @@ IMAGE_TOKEN = "<|imgpad|>"
class DotsOCRImagePixelInputs(TensorSchema):
"""
Dimensions:
- np: The total number of patches over each image over each prompt in
the batch
- ni: Number of images
- cps: Number of channels * patch_size * patch_size
"""Dimensions:
- np: The total number of patches over each image over each prompt in
the batch
- ni: Number of images
- cps: Number of channels * patch_size * patch_size
"""
type: Literal["pixel_values"]
@@ -81,11 +80,10 @@ class DotsOCRImagePixelInputs(TensorSchema):
class DotsOCRImageEmbeddingInputs(TensorSchema):
"""
Dimensions:
- nf: Number of image features
- hs: Hidden size
- ni: Number of images
"""Dimensions:
- nf: Number of image features
- hs: Hidden size
- ni: Number of images
"""
type: Literal["image_embeds"]
@@ -768,9 +766,7 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def get_mm_mapping(self) -> MultiModelKeys:
"""
Get the module prefix in multimodal models
"""
"""Get the module prefix in multimodal models."""
return MultiModelKeys.from_string_field(
language_model="language_model",
connector="vision_tower.merger",
+15 -18
View File
@@ -37,13 +37,12 @@ from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
class Eagle2_5_VLImagePixelInputs(TensorSchema):
"""
Dimensions:
- bn: Batch size * number of images
- bnp: Batch size * number of images * (1 + num_patches)
- c: Number of channels (3)
- h: Height of each image patch
- w: Width of each image patch
"""Dimensions:
- bn: Batch size * number of images
- bnp: Batch size * number of images * (1 + num_patches)
- c: Number of channels (3)
- h: Height of each image patch
- w: Width of each image patch
"""
type: Literal["pixel_values"]
@@ -52,11 +51,10 @@ class Eagle2_5_VLImagePixelInputs(TensorSchema):
class Eagle2_5_VLImageEmbeddingInputs(TensorSchema):
"""
Dimensions:
- n: Number of images
- f: Total image feature size
- h: Hidden size (must match the hidden size of language model backbone)
"""Dimensions:
- n: Number of images
- f: Total image feature size
- h: Hidden size (must match the hidden size of language model backbone)
"""
type: Literal["image_embeds"]
@@ -127,8 +125,7 @@ class Eagle2_5_VLMultiModalProcessor(
class Eagle2_5_VLForConditionalGeneration(
nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA
):
"""
Eagle2.5-VL model for conditional generation.
"""Eagle2.5-VL model for conditional generation.
Architecture:
- Vision Encoder: SigLIP
@@ -238,8 +235,7 @@ class Eagle2_5_VLForConditionalGeneration(
)
def pixel_shuffle(self, x: torch.Tensor, scale_factor: float = 0.5) -> torch.Tensor:
"""
Pixel shuffle operation for downsampling vision features.
"""Pixel shuffle operation for downsampling vision features.
Args:
x: Input tensor of shape (n, w, h, c)
@@ -247,6 +243,7 @@ class Eagle2_5_VLForConditionalGeneration(
Returns:
Downsampled tensor
"""
n, w, h, c = x.size()
# N, W, H, C --> N, W, H * scale, C // scale
@@ -264,14 +261,14 @@ class Eagle2_5_VLForConditionalGeneration(
return x
def extract_feature(self, pixel_values: torch.Tensor) -> torch.Tensor:
"""
Extract visual features from pixel values.
"""Extract visual features from pixel values.
Args:
pixel_values: Input pixel values of shape (batch, channels, height, width)
Returns:
Visual embeddings
"""
vit_embeds = self.vision_model(pixel_values=pixel_values)
+13 -14
View File
@@ -119,7 +119,7 @@ def all_gather_interleave(local_tensor, hidden_size: int, tp_size: int):
class Ernie4_5_VisionAttention(nn.Module):
"""VisionAttention using VLLM framework APIs"""
"""VisionAttention using VLLM framework APIs."""
def __init__(
self,
@@ -497,6 +497,7 @@ class Ernie4_5_VisionTransformer(nn.Module):
Returns:
Dict with ``rotary_pos_emb``, ``cu_seqlens`` and ``max_seqlen``.
"""
if device is None:
device = self.device
@@ -600,12 +601,11 @@ class Ernie4_5_VisionTransformer(nn.Module):
class Ernie4_5_VLImagePixelInputs(TensorSchema):
"""
Dimensions:
- np: The total number of patches over each image over each prompt in
the batch
- ni: Number of images
- cps: Number of channels * patch_size * patch_size
"""Dimensions:
- np: The total number of patches over each image over each prompt in
the batch
- ni: Number of images
- cps: Number of channels * patch_size * patch_size
"""
type: Literal["pixel_values"]
@@ -618,13 +618,12 @@ Ernie4_5_VLImageInputs = Ernie4_5_VLImagePixelInputs
class Ernie4_5_VLVideoPixelInputs(TensorSchema):
"""
Dimensions:
- np: The total number of patches over each image over each prompt in
the batch
- ni: Number of images
- cps: Number of channels * temporal_patch_size * patch_size *
patch_size
"""Dimensions:
- np: The total number of patches over each image over each prompt in
the batch
- ni: Number of images
- cps: Number of channels * temporal_patch_size * patch_size *
patch_size
"""
type: Literal["pixel_values_videos"]
@@ -45,8 +45,7 @@ def unified_kv_cache_update(
to_cache: torch.Tensor,
layer_name: str,
) -> torch.Tensor:
"""
Returns a dummy that is passed to unified_attention to signal a side effect and
"""Returns a dummy that is passed to unified_attention to signal a side effect and
the data dependency between them to ensure torch.compile preserves ordering.
"""
forward_context = get_forward_context()
@@ -294,6 +293,7 @@ class CacheOnlyAttentionLayer(nn.Module, AttentionLayerBase):
Returns:
Dummy output tensor (not used)
"""
# Note: we set num_heads to num_hidden_layers and
# head_size to hidden_size for hidden states storage
@@ -373,8 +373,8 @@ class ExtractHiddenStatesModel(nn.Module):
Returns:
Tuple of (dummy_output, dummy_output) - both unused
"""
"""
# Call dummy attention layer to cache hidden states
# Output is ignored - we only care about the KV cache side effects
_ = self.cache_only_layers[str(self.target_num_hidden_layers)](hidden_states)
+3 -4
View File
@@ -140,8 +140,7 @@ class FalconH1SSMDecoderLayer(nn.Module):
self._init_mup_vector()
def _init_mup_vector(self):
"""
Non learnable per-block scaling vector composed of element-wise
"""Non learnable per-block scaling vector composed of element-wise
multipliersapplied to each separate contiguous block of the output
of the linear projection (in_proj) before further processing
(gating, convolution, SSM):
@@ -313,8 +312,7 @@ class FalconH1AttentionDecoderLayer(nn.Module):
class FalconH1ParallelHybrid(nn.Module):
"""
A hybrid decoder layer for FalconH1 where the input is processed
"""A hybrid decoder layer for FalconH1 where the input is processed
in parallel through both the self-attention branch and the SSM (Mamba)
branch. Their outputs are then summed to produce the final hidden state.
@@ -552,6 +550,7 @@ class FalconH1ForCausalLM(
Tuple containing:
- conv_state_shape: Shape for convolutional state cache
- temporal_state_shape: Shape for state space model cache
"""
parallel_config = vllm_config.parallel_config
hf_config = vllm_config.model_config.hf_config
+4 -5
View File
@@ -63,11 +63,10 @@ logger = init_logger(__name__)
class FireRedASR2AudioInputs(TensorSchema):
"""
Dimensions:
- b: Batch size
- nmb: Number of mel bins
- t: Time frames (M)
"""Dimensions:
- b: Batch size
- nmb: Number of mel bins
- t: Time frames (M)
"""
input_features: Annotated[
+4 -5
View File
@@ -598,11 +598,10 @@ class Transformer(nn.Module):
class FunASRAudioInputs(TensorSchema):
"""
Dimensions:
- b: Batch size
- nmb: Number of mel bins
- t: Time frames (M)
"""Dimensions:
- b: Batch size
- nmb: Number of mel bins
- t: Time frames (M)
"""
input_features: Annotated[
+8 -11
View File
@@ -58,14 +58,13 @@ logger = init_logger(__name__)
class Gemma3ImagePixelInputs(TensorSchema):
"""
Dimensions:
- p: Number of patches total (over each image over each prompt in the
batch)
- c: Number of channels (3)
- h: Height of each patch
- w: Width of each patch
- bn: Batch size * number of images
"""Dimensions:
- p: Number of patches total (over each image over each prompt in the
batch)
- c: Number of channels (3)
- h: Height of each patch
- w: Width of each patch
- bn: Batch size * number of images
"""
type: Literal["pixel_values"] = "pixel_values"
@@ -678,9 +677,7 @@ class Gemma3ForConditionalGeneration(
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def get_mm_mapping(self) -> MultiModelKeys:
"""
Get the module prefix in multimodal models
"""
"""Get the module prefix in multimodal models"""
return MultiModelKeys.from_string_field(
language_model="language_model",
connector="multi_modal_projector",
+5 -8
View File
@@ -185,7 +185,7 @@ class Gemma3nAltUp(nn.Module):
class Gemma3nLaurelBlock(nn.Module):
"""Learned Augmented Residual Layer"""
"""Learned Augmented Residual Layer."""
def __init__(
self,
@@ -593,9 +593,7 @@ class Gemma3nDecoderLayer(nn.Module):
enable_if=lambda vllm_config: vllm_config.cache_config.kv_sharing_fast_prefill
)
class Gemma3nSelfDecoder(nn.Module):
"""
Includes altup embedding and self decoder layers
"""
"""Includes altup embedding and self decoder layers."""
def __init__(
self,
@@ -776,9 +774,7 @@ class Gemma3nSelfDecoder(nn.Module):
enable_if=lambda vllm_config: vllm_config.cache_config.kv_sharing_fast_prefill
)
class Gemma3nCrossDecoder(nn.Module):
"""
Cross-decoder layers
"""
"""Cross-decoder layers."""
def __init__(
self,
@@ -818,7 +814,8 @@ class Gemma3nCrossDecoder(nn.Module):
def _kv_sharing_weights_mapper(config: Gemma3nTextConfig) -> WeightsMapper:
"""KV-shared layers only have q_proj, so qkv_proj packing applies to the
other layers. Original checkpoints still ship K/V tensors for the shared
layers (fine-tuned ones omit them); those are dropped."""
layers (fine-tuned ones omit them); those are dropped.
"""
first_kv_shared_layer_idx = config.num_hidden_layers - config.num_kv_shared_layers
return WeightsMapper(
orig_to_new_substr={
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Lightweight utility functions for Gemma3n audio processing.
"""Lightweight utility functions for Gemma3n audio processing.
This module is separate from gemma3n_mm.py to avoid heavy CUDA dependencies,
making it testable without a full vLLM build.
@@ -37,6 +36,7 @@ def adjust_audio_features_to_expected_length(
Tuple of:
- adjusted_features: Audio features adjusted to expected_tokens length
- tokens_truncated: Number of tokens truncated (0 if padding was applied)
"""
audio_batch_size, audio_seq_len, audio_embed_dim = audio_features.shape
tokens_truncated = 0
+18 -23
View File
@@ -74,12 +74,11 @@ TOKENS_PER_AUDIO = 188
class Gemma3nImagePixelInputs(TensorSchema):
"""
Dimensions:
- bn: Batch size * number of images
- c: Number of channels (3)
- h: Height of each patch
- w: Width of each patch
"""Dimensions:
- bn: Batch size * number of images
- c: Number of channels (3)
- h: Height of each patch
- w: Width of each patch
"""
type: Literal["pixel_values"] = "pixel_values"
@@ -113,6 +112,7 @@ def batch_audio_features(
Returns:
The `(bn, s_max, f)` features and their `(bn, s_max)` mask.
"""
if isinstance(input_features, torch.Tensor):
assert isinstance(input_features_mask, torch.Tensor)
@@ -135,11 +135,10 @@ def batch_audio_features(
class Gemma3nAudioInputs(TensorSchema):
"""
Dimensions:
- bn: Batch size * number of audios
- s: seq_length
- f: num_features
"""Dimensions:
- bn: Batch size * number of audios
- s: seq_length
- f: num_features
"""
type: Literal["audio"] = "audio"
@@ -187,8 +186,7 @@ class Gemma3nProcessingInfo(BaseProcessingInfo):
image_height: int,
processor: Gemma3nProcessor,
) -> PromptUpdateDetails:
"""
Get the replacement metadata for image tokens.
"""Get the replacement metadata for image tokens.
For Gemma3n, this should return the full_image_sequence which includes
BOI token, repeated image tokens, and EOI token.
@@ -205,8 +203,7 @@ class Gemma3nProcessingInfo(BaseProcessingInfo):
*,
processor: Gemma3nProcessor,
) -> PromptUpdateDetails:
"""
Get the replacement metadata for audio tokens.
"""Get the replacement metadata for audio tokens.
For Gemma3n, this should return the full_audio_sequence which includes
BOA token, repeated audio tokens, and EOA token.
@@ -316,8 +313,7 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
return processed_data
def _get_num_audio_frames_alone(self, num_samples: int) -> int:
"""
Get the number of mel frames that the feature extractor produces
"""Get the number of mel frames that the feature extractor produces
for a single audio clip of the given length, independent of batch
padding.
"""
@@ -514,7 +510,8 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
class Gemma3nMultimodalEmbedder(nn.Module):
"""Embeds token ids or soft tokens for multimodal content into language
model space."""
model space.
"""
def __init__(
self,
@@ -571,6 +568,7 @@ class Gemma3nMultimodalEmbedder(nn.Module):
Returns:
A torch.Tensor of embeddings with shape `[batch_size, seq_len, self.config.text_config.hidden_size]`.
""" # noqa: E501
if (input_ids is None) ^ (inputs_embeds is not None):
raise ValueError(
@@ -891,9 +889,7 @@ class Gemma3nForConditionalGeneration(
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def get_mm_mapping(self) -> MultiModelKeys:
"""
Get the module prefix in multimodal models
"""
"""Get the module prefix in multimodal models"""
return MultiModelKeys.from_string_field(
language_model="language_model",
connector="multi_modal_projector",
@@ -911,8 +907,7 @@ class Gemma3nForConditionalGeneration(
@classmethod
def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType:
"""
Gemma3n supports "free-form" transcription.
"""Gemma3n supports "free-form" transcription.
We fix its prompt here to standardize transcriptions/translations
requests.
"""
+2
View File
@@ -866,6 +866,7 @@ class Gemma4SelfDecoderLayers(nn.Module):
Returns:
Per-layer embeddings (num_tokens, num_layers,
hidden_size_per_layer_input)
"""
if self.embed_tokens_per_layer is None:
return None
@@ -1199,6 +1200,7 @@ class Gemma4Model(nn.Module, EagleModelMixin):
Returns:
Per-layer embeddings (num_tokens, num_layers,
hidden_size_per_layer_input)
"""
return self.self_decoder.get_per_layer_inputs(input_ids)
+12 -7
View File
@@ -132,8 +132,7 @@ def _get_max_soft_tokens(
class Gemma4ImagePixelInputs(TensorSchema):
"""
Pre-patchified image inputs from the Gemma4 image processor.
"""Pre-patchified image inputs from the Gemma4 image processor.
Dimensions:
- bn: Batch size * number of images
@@ -159,11 +158,10 @@ class Gemma4ImagePixelInputs(TensorSchema):
class Gemma4AudioInputs(TensorSchema):
"""
Dimensions:
- bn: Batch size * number of audios
- s: Sequence length (MEL spectrogram frames)
- f: Number of features (MEL bins)
"""Dimensions:
- bn: Batch size * number of audios
- s: Sequence length (MEL spectrogram frames)
- f: Number of features (MEL bins)
"""
type: Literal["audio"] = "audio"
@@ -304,9 +302,12 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
for an image of the given dimensions, after padding is stripped.
Args:
image_width: Width of the image in pixels.
image_height: Height of the image in pixels.
max_soft_tokens: Override for the vision config's
``default_output_length``. When *None*, the value from
the model config is used.
"""
vision_cfg = self.get_hf_config().vision_config
patch_size = vision_cfg.patch_size
@@ -345,8 +346,12 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
produce after stripping padding.
Args:
image_width: Width of the image in pixels.
image_height: Height of the image in pixels.
processor: The Gemma4 processor, when already constructed.
max_soft_tokens: Override for the default token budget.
When *None*, falls back to the model config value.
"""
if processor is None:
processor = self.get_hf_processor()
+1
View File
@@ -101,6 +101,7 @@ class Gemma4MTPMaskedEmbedder(nn.Module):
Returns:
logits: (num_tokens, num_selected) sparse logits.
indices: (num_tokens, num_selected) corresponding vocab indices.
"""
num_tokens = hidden_states.shape[0]
_, top_k_indices = torch.topk(
+28 -32
View File
@@ -25,7 +25,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Inference-only GLM-4.1V & GLM-4.6V-Flash, AutoGLM-Phone-9B model
compatible with HuggingFace weights."""
compatible with HuggingFace weights.
"""
import math
from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence
@@ -145,12 +146,11 @@ def _to_video_metadata(metadata: Mapping[str, Any]) -> VideoMetadata:
class Glm4vImagePixelInputs(TensorSchema):
"""
Dimensions:
- np: Number of patches
- cpp: Number of channels * patch_size * patch_size
- ni: Number of images
- g: Grid dimensions (3 for grid_t, grid_h, grid_w)
"""Dimensions:
- np: Number of patches
- cpp: Number of channels * patch_size * patch_size
- ni: Number of images
- g: Grid dimensions (3 for grid_t, grid_h, grid_w)
"""
type: Literal["pixel_values"] = "pixel_values"
@@ -160,12 +160,11 @@ class Glm4vImagePixelInputs(TensorSchema):
class Glm4vImageEmbeddingInputs(TensorSchema):
"""
Dimensions:
- f: Number of image features (varies based on image resolution)
- h: Hidden size (must match language model backbone)
- n: Number of images
- g: Grid dimensions (3 for grid_t, grid_h, grid_w)
"""Dimensions:
- f: Number of image features (varies based on image resolution)
- h: Hidden size (must match language model backbone)
- n: Number of images
- g: Grid dimensions (3 for grid_t, grid_h, grid_w)
"""
type: Literal["image_embeds"] = "image_embeds"
@@ -178,14 +177,13 @@ Glm4vImageInputs: TypeAlias = Glm4vImagePixelInputs | Glm4vImageEmbeddingInputs
class Glm4vVideoPixelInputs(TensorSchema):
"""
Dimensions:
- np: Number of patches
- ctpp: Number of channels * temporal_patch_size *
patch_size * patch_size
- f: Number of frames
- g: Grid dimensions (3 for grid_t which is usually 1 for processed
video, grid_h, grid_w)
"""Dimensions:
- np: Number of patches
- ctpp: Number of channels * temporal_patch_size *
patch_size * patch_size
- f: Number of frames
- g: Grid dimensions (3 for grid_t which is usually 1 for processed
video, grid_h, grid_w)
"""
type: Literal["pixel_values_videos"] = "pixel_values_videos"
@@ -195,13 +193,12 @@ class Glm4vVideoPixelInputs(TensorSchema):
class Glm4vVideoEmbeddingInputs(TensorSchema):
"""
Dimensions:
- p: Number of video patches across all frames
- h: Hidden size (must match language model backbone)
- f: Number of frames
- g: Grid dimensions (3 for grid_t which is usually 1 for processed
video, grid_h, grid_w)
"""Dimensions:
- p: Number of video patches across all frames
- h: Hidden size (must match language model backbone)
- f: Number of frames
- g: Grid dimensions (3 for grid_t which is usually 1 for processed
video, grid_h, grid_w)
"""
type: Literal["video_embeds"] = "video_embeds"
@@ -771,7 +768,6 @@ class Glm4vVisionTransformer(nn.Module):
forward graph compatible with CUDA graph replay. Coordinate generation matches
`rot_pos_emb` exactly to guarantee spatial alignment.
"""
device = self.embeddings.position_embedding.weight.device
dtype = self.dtype
all_embeds = []
@@ -850,6 +846,7 @@ class Glm4vVisionTransformer(nn.Module):
instead of computing from cu_seqlens (needed for CUDA
graph capture to cover worst-case replay scenarios).
device: Device to place tensors on. Defaults to self.device.
"""
if device is None:
device = self.device
@@ -2347,6 +2344,7 @@ class Glm4vForConditionalGeneration(
parallelism.
inputs_embeds: Optional pre-computed input embeddings.
**kwargs: Additional keyword arguments.
"""
if intermediate_tensors is not None:
inputs_embeds = None
@@ -2370,9 +2368,7 @@ class Glm4vForConditionalGeneration(
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def get_mm_mapping(self) -> MultiModelKeys:
"""
Get the module prefix in multimodal models
"""
"""Get the module prefix in multimodal models."""
return MultiModelKeys.from_string_field(
language_model="language_model.model",
connector="visual.merger.",
+2 -1
View File
@@ -22,7 +22,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Inference-only GLM-4.5, GLM-4.6, GLM-4.7 model
compatible with HuggingFace weights."""
compatible with HuggingFace weights.
"""
from collections.abc import Iterable
from itertools import islice
@@ -454,8 +454,7 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
return loaded_params
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
"""
Rewrite the weight name to match the format of the original model.
"""Rewrite the weight name to match the format of the original model.
Add .mtp_block for modules in transformer layer block for spec layer
and rename shared layer weights to be top level.
"""
+3 -3
View File
@@ -22,7 +22,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Inference-only GLM-4.5, GLM-4.6, GLM-4.7 MTP
model compatible with HuggingFace weights."""
model compatible with HuggingFace weights.
"""
import typing
from collections.abc import Callable, Iterable
@@ -412,8 +413,7 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts):
return loaded_params
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
"""
Rewrite the weight name to match the format of the original model.
"""Rewrite the weight name to match the format of the original model.
Add .mtp_block for modules in transformer layer block for spec layer
and rename shared layer weights to be top level.
"""
+15 -17
View File
@@ -65,12 +65,11 @@ from .utils import WeightsMapper
class GLMVImagePixelInputs(TensorSchema):
"""
Dimensions:
- b: Batch size
- c: Number of channels (3)
- h: Height of image
- w: Width of image
"""Dimensions:
- b: Batch size
- c: Number of channels (3)
- h: Height of image
- w: Width of image
"""
type: Literal["pixel_values"] = "pixel_values"
@@ -90,14 +89,15 @@ class EVA2CLIPPatchEmbedding(nn.Module):
self.position_embedding = nn.Embedding(config.num_positions, config.hidden_size)
def forward(self, images: torch.Tensor) -> torch.Tensor:
"""
Parameters:
"""Parameters
images : torch.Tensor
Input image tensor with shape (B, C, H, W)
Returns:
Returns
-------
torch.Tensor
Transformed tensor with shape (B, L, D)
"""
images = images.to(device=self.proj.weight.device, dtype=self.proj.weight.dtype)
x = self.proj(images)
@@ -246,8 +246,7 @@ class EVA2CLIPGLU(nn.Module):
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
"""
The original implementation is the same as:
"""The original implementation is the same as:
```python
self.dense_h_to_4h = ColumnParallelLinear(
config.hidden_size,
@@ -349,14 +348,15 @@ class EVA2CLIPModel(nn.Module):
self.scaling_factor = vision_config.scaling_factor
def forward(self, images: torch.Tensor) -> torch.Tensor:
"""
Parameters:
"""Parameters
images : torch.Tensor
Input image tensor with shape (B, C, H, W)
Returns:
Returns
-------
torch.Tensor
Transformed tensor with shape (B, L, D)
"""
x = self.patch_embedding(images)
x = self.transformer(x)
@@ -518,9 +518,7 @@ class GLM4VForCausalLM(
}
def get_mm_mapping(self) -> MultiModelKeys:
"""
Get the module prefix in multimodal models
"""
"""Get the module prefix in multimodal models."""
return MultiModelKeys.from_string_field(
language_model="transformer.encoder",
connector="transformer.vision.linear_proj",
+1 -2
View File
@@ -246,8 +246,7 @@ class GlmOcrMTP(nn.Module, SupportsPP):
return loaded_params
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
"""
Rewrite the weight name to match the format of the original model.
"""Rewrite the weight name to match the format of the original model.
Add .mtp_block for modules in transformer layer block for spec layer
and rename shared layer weights to be top level.
"""
+33 -45
View File
@@ -75,8 +75,7 @@ from .whisper import ISO639_1_SUPPORTED_LANGS, _create_fake_bias_for_k_proj
class GlmAsrEncoderRotaryEmbedding(nn.Module):
"""
Rotary Position Embedding for GLM-ASR encoder.
"""Rotary Position Embedding for GLM-ASR encoder.
Computes rotary position embeddings on-demand for efficiency.
Only caches inv_freq as a buffer; cos/sin are computed during forward
@@ -115,8 +114,7 @@ class GlmAsrEncoderRotaryEmbedding(nn.Module):
self.register_buffer("inv_freq", inv_freq, persistent=False)
def forward(self, seq_len: int) -> torch.Tensor:
"""
Compute rotary position frequencies for given sequence length.
"""Compute rotary position frequencies for given sequence length.
Args:
seq_len: The sequence length to compute embeddings for.
@@ -124,6 +122,7 @@ class GlmAsrEncoderRotaryEmbedding(nn.Module):
Returns:
Frequency tensor with shape [seq_len, dim/2]. Use .cos() and
.sin() to get the rotary embedding components.
"""
# Compute on the same device as inv_freq (automatically correct after .to())
seq = torch.arange(
@@ -134,8 +133,7 @@ class GlmAsrEncoderRotaryEmbedding(nn.Module):
class GlmAsrEncoderAttention(nn.Module):
"""
Optimized Multi-headed Grouped Query Attention for GLM-ASR encoder.
"""Optimized Multi-headed Grouped Query Attention for GLM-ASR encoder.
Uses vLLM's QKVParallelLinear for fused projections, ApplyRotaryEmb for
rotary position embeddings, and MMEncoderAttention for hardware-optimized
@@ -208,14 +206,14 @@ class GlmAsrEncoderAttention(nn.Module):
rotary_pos_emb_cos: torch.Tensor,
rotary_pos_emb_sin: torch.Tensor,
) -> torch.Tensor:
"""
Args:
"""Args:
hidden_states: [batch_size, seq_len, hidden_size]
rotary_pos_emb_cos: [seq_len, rotary_dim/2] - cosine of rotary embeddings
rotary_pos_emb_sin: [seq_len, rotary_dim/2] - sine of rotary embeddings
Returns:
[batch_size, seq_len, hidden_size]
"""
batch_size, seq_len, _ = hidden_states.shape
@@ -255,8 +253,7 @@ class GlmAsrEncoderAttention(nn.Module):
class GlmAsrEncoderMLP(nn.Module):
"""
Optimized MLP for GLM-ASR encoder.
"""Optimized MLP for GLM-ASR encoder.
Uses vLLM's parallel linear layers for better performance.
"""
@@ -297,8 +294,7 @@ class GlmAsrEncoderMLP(nn.Module):
class GlmAsrEncoderLayer(nn.Module):
"""
Optimized Transformer encoder layer for GLM-ASR.
"""Optimized Transformer encoder layer for GLM-ASR.
Combines attention and MLP with residual connections and layer norms.
"""
@@ -335,14 +331,14 @@ class GlmAsrEncoderLayer(nn.Module):
rotary_pos_emb_cos: torch.Tensor,
rotary_pos_emb_sin: torch.Tensor,
) -> torch.Tensor:
"""
Args:
"""Args:
hidden_states: [batch_size, seq_len, hidden_size]
rotary_pos_emb_cos: [seq_len, rotary_dim/2] - cosine of rotary embeddings
rotary_pos_emb_sin: [seq_len, rotary_dim/2] - sine of rotary embeddings
Returns:
[batch_size, seq_len, hidden_size]
"""
# Self-attention with residual
residual = hidden_states
@@ -364,8 +360,7 @@ class GlmAsrEncoderLayer(nn.Module):
class _GlmAsrEncoderOutput:
"""
Simple output container compatible with transformers' BaseModelOutput.
"""Simple output container compatible with transformers' BaseModelOutput.
This lightweight container holds the encoder output and is compatible
with the transformers library's output format while being more efficient
@@ -374,6 +369,7 @@ class _GlmAsrEncoderOutput:
Attributes:
last_hidden_state: Final layer hidden states from the encoder.
Shape: [batch_size, seq_len, hidden_size]
"""
__slots__ = ("last_hidden_state",)
@@ -383,8 +379,7 @@ class _GlmAsrEncoderOutput:
class GlmAsrEncoder(nn.Module):
"""
Optimized GLM-ASR Audio Encoder with vLLM native implementation.
"""Optimized GLM-ASR Audio Encoder with vLLM native implementation.
This encoder processes audio features through convolutional layers
followed by transformer layers with rotary position embeddings.
@@ -454,14 +449,14 @@ class GlmAsrEncoder(nn.Module):
def _get_feat_extract_output_lengths(
self, input_lengths: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Compute the output length after convolutions.
"""Compute the output length after convolutions.
Args:
input_lengths: Input sequence lengths [batch_size]
Returns:
Tuple of (output after conv1, output after conv2)
"""
# Conv1: kernel=3, stride=1, padding=1
output_lengths_conv1 = (input_lengths + 2 * 1 - 3) // 1 + 1
@@ -472,8 +467,7 @@ class GlmAsrEncoder(nn.Module):
return output_lengths_conv1, output_lengths_conv2
def forward(self, input_features: torch.Tensor) -> _GlmAsrEncoderOutput:
"""
Forward pass through the encoder.
"""Forward pass through the encoder.
Args:
input_features: [batch_size, num_mel_bins, seq_len]
@@ -482,6 +476,7 @@ class GlmAsrEncoder(nn.Module):
_GlmAsrEncoderOutput: Object with .last_hidden_state attribute \
containing [batch_size, seq_len', hidden_size] where seq_len' \
is the sequence length after convolutions
"""
# Apply convolutional layers with GELU activation
hidden_states = torch.nn.functional.gelu(self.conv1(input_features))
@@ -515,11 +510,10 @@ class GlmAsrEncoder(nn.Module):
class GlmAsrFeatureInputs(TensorSchema):
"""
Dimensions:
- num_chunks: Number of audio chunks (flattened)
- nmb: Number of mel bins
- num_audios: Number of original audio files
"""Dimensions:
- num_chunks: Number of audio chunks (flattened)
- nmb: Number of mel bins
- num_audios: Number of original audio files
"""
type: Literal["audio_features"]
@@ -538,12 +532,11 @@ class GlmAsrFeatureInputs(TensorSchema):
class GlmAsrEmbeddingInputs(TensorSchema):
"""
Dimensions:
- bn: Batch size
- naf: Number of audio features
- hs: Hidden size (must match the hidden size of language model
backbone)
"""Dimensions:
- bn: Batch size
- naf: Number of audio features
- hs: Hidden size (must match the hidden size of language model
backbone)
"""
type: Literal["audio_embeds"] = "audio_embeds"
@@ -557,8 +550,7 @@ GlmAsrInputs: TypeAlias = GlmAsrFeatureInputs | GlmAsrEmbeddingInputs
class GlmAsrMultiModalProjector(nn.Module):
"""
Projects audio encoder outputs to language model hidden space.
"""Projects audio encoder outputs to language model hidden space.
This projector uses a two-layer MLP to map audio features from the
encoder's intermediate size to the language model's hidden size.
@@ -601,8 +593,7 @@ class GlmAsrMultiModalProjector(nn.Module):
def _glmasr_field_config(
hf_inputs: Mapping[str, torch.Tensor],
) -> dict[str, MultiModalFieldConfig]:
"""
Configure multimodal field batching strategy for GLM-ASR.
"""Configure multimodal field batching strategy for GLM-ASR.
Determines how to batch audio inputs based on whether chunking is used.
When chunk_counts is present, features are flattened across chunks;
@@ -614,6 +605,7 @@ def _glmasr_field_config(
Returns:
Dictionary mapping field names to MultiModalFieldConfig objects \
that specify batching behavior.
"""
chunk_counts = hf_inputs.get("chunk_counts")
if chunk_counts is not None:
@@ -636,8 +628,7 @@ def _glmasr_field_config(
class GlmAsrMultiModalDataParser(MultiModalDataParser):
"""
Custom parser for GLM-ASR multimodal data.
"""Custom parser for GLM-ASR multimodal data.
Extends the base parser to handle GLM-ASR specific audio data formats,
including both pre-computed audio embeddings and raw audio features.
@@ -658,8 +649,7 @@ class GlmAsrMultiModalDataParser(MultiModalDataParser):
class GlmAsrProcessingInfo(BaseProcessingInfo):
"""
Processing information provider for GLM-ASR model.
"""Processing information provider for GLM-ASR model.
Provides access to model configuration, processor, and feature extractor
needed for audio preprocessing and multimodal integration.
@@ -687,8 +677,7 @@ class GlmAsrProcessingInfo(BaseProcessingInfo):
class GlmAsrDummyInputsBuilder(BaseDummyInputsBuilder[GlmAsrProcessingInfo]):
"""
Builder for dummy inputs used in profiling and testing.
"""Builder for dummy inputs used in profiling and testing.
Generates dummy text prompts and audio data that match the expected
format for GLM-ASR model inputs. Used for memory profiling and
@@ -727,8 +716,7 @@ class GlmAsrDummyInputsBuilder(BaseDummyInputsBuilder[GlmAsrProcessingInfo]):
class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"]):
"""
GLM-ASR processor that inherits directly from BaseMultiModalProcessor
"""GLM-ASR processor that inherits directly from BaseMultiModalProcessor
for better performance and cleaner implementation.
"""
+2 -2
View File
@@ -71,8 +71,7 @@ def _get_audio_output_lengths_for_tower(
merge_factor: int,
conv_params: list[tuple[int, int, int]],
) -> torch.Tensor:
"""
Calculate the output lengths after audio processing.
"""Calculate the output lengths after audio processing.
The output length accounts for:
1. Convolution layers (downsampling)
@@ -86,6 +85,7 @@ def _get_audio_output_lengths_for_tower(
Returns:
Output lengths after all processing [batch_size]
"""
# First, calculate the output length after convolutions
if hasattr(audio_tower, "_get_feat_extract_output_lengths"):
+1
View File
@@ -327,6 +327,7 @@ class GPT2ForSequenceClassification(nn.Module, SupportsCrossEncoding):
transformer: An instance of GPT2Model used for forward operations.
score: A layer for calculating logits.
_pooler: An instance of Pooler used for pooling operations.
"""
is_pooling_model = True
+1
View File
@@ -790,6 +790,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
Returns:
Weight dtype string (e.g., "mxfp4", "fp8") or None if not available
"""
if hasattr(self.layers[layer_id].mlp.experts._quant_method, "weight_dtype"):
return self.layers[layer_id].mlp.experts._quant_method.weight_dtype
+1
View File
@@ -84,6 +84,7 @@ def granite_layer_attn_params(
Returns:
Sliding window size (`None` for full attention), RoPE base theta (`0`
for NoPE), and attention sink presence/absence.
"""
layer_types = getattr(config, "layer_types", None)
sliding_window = (
@@ -684,6 +684,7 @@ class Granite4VisionForConditionalGeneration(
Packing on dim=-1 means the framework's token-level slicing for
chunked prefill preserves all levels intact.
"""
select_strategy = self._vision_feature_select_strategy
+9 -3
View File
@@ -91,8 +91,7 @@ ISO639_1_SUPPORTED_LANGS = {
### Audio Input
class GraniteSpeechAudioInputs(TensorSchema):
"""
Audio input features for Granite Speech model.
"""Audio input features for Granite Speech model.
Dimensions:
- b: Batch size
@@ -477,7 +476,8 @@ class GraniteSpeechConformerConvModule(nn.Module):
class GraniteSpeechConformerBlock(nn.Module):
"""Conformer block, consisting largely of linear layers,
attention, and convolutional layers."""
attention, and convolutional layers.
"""
def __init__(self, config: PretrainedConfig, prefix: str = ""):
super().__init__()
@@ -732,9 +732,11 @@ class GraniteSpeechForConditionalGeneration(
Args:
audio_embed_sizes: torch.Tensor
Tensor of num features in each seq in the batch.
Returns:
torch.Tensor: Mask of shape (bsz, num_features) to be applied to
the audio features prior to splitting the audio embeddings.
"""
most_audio_features = int(torch.max(audio_embed_sizes))
mask_indices = torch.arange(most_audio_features).view(1, -1)
@@ -763,10 +765,12 @@ class GraniteSpeechForConditionalGeneration(
Args:
input_features: list[torch.Tensor]
3D Input features to be coerced into a tensor.
Returns:
torch.Tensor: Tensor of shape [bsz, num_features, 160], where
num_features is the max number of features of any entry in the
batch.
"""
feat_lens = [feats.shape[1] for feats in input_features]
padding = [max(feat_lens) - length for length in feat_lens]
@@ -790,8 +794,10 @@ class GraniteSpeechForConditionalGeneration(
audio_input: GraniteSpeechAudioInputs
Audio inputs object containing Mel features, an input features
mask, and the (flattened) number of audio tokens per instance.
Returns:
tuple[torch.Tensor]: List of length bsz.
"""
# TODO (Alex) - support embedding inputs
encoder_embeds = self.encoder(audio_input["input_features"])
@@ -633,6 +633,7 @@ class GraniteMoeHybridForCausalLM(
Tuple containing:
- conv_state_shape: Shape for convolutional state cache
- temporal_state_shape: Shape for state space model cache
"""
parallel_config = vllm_config.parallel_config
hf_config = vllm_config.model_config.hf_config
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
HRM-Text: Hierarchical Reasoning Model — Text variant.
"""HRM-Text: Hierarchical Reasoning Model — Text variant.
Reference Hugging Face implementation:
src/transformers/models/hrm_text/modeling_hrm_text.py
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
HyperCLOVAX V2 (32B Think Model) Implementation.
"""HyperCLOVAX V2 (32B Think Model) Implementation.
This module contains the V2 architecture that uses Qwen2.5 Vision Transformer
instead of CLIP/SigLIP used in V1.
@@ -57,8 +56,7 @@ V2_VIDEO_TOKEN: str = "<|video_start|><|VIDEO_PAD|><|video_end|>"
class HCXVisionV2ImagePixelInputs(TensorSchema):
"""
V2 Image inputs using Qwen2.5-VL style grid_thw format.
"""V2 Image inputs using Qwen2.5-VL style grid_thw format.
Dimensions:
- np: Number of patches
@@ -72,8 +70,7 @@ class HCXVisionV2ImagePixelInputs(TensorSchema):
class HCXVisionV2ImageEmbeddingInputs(TensorSchema):
"""
V2 Image embedding inputs.
"""V2 Image embedding inputs.
Dimensions:
- nf: Number of image features
@@ -90,8 +87,7 @@ HCXVisionV2ImageInputs = HCXVisionV2ImagePixelInputs | HCXVisionV2ImageEmbedding
class HCXVisionV2VideoPixelInputs(TensorSchema):
"""
V2 Video inputs using Qwen2.5-VL style grid_thw format.
"""V2 Video inputs using Qwen2.5-VL style grid_thw format.
Dimensions:
- np: Number of patches
@@ -105,8 +101,7 @@ class HCXVisionV2VideoPixelInputs(TensorSchema):
class HCXVisionV2VideoEmbeddingInputs(TensorSchema):
"""
V2 Video embedding inputs.
"""V2 Video embedding inputs.
Dimensions:
- nf: Number of video features
@@ -419,8 +414,7 @@ class HCXVisionV2MultiModalProcessor(
dummy_inputs=HCXVisionV2DummyInputsBuilder,
)
class HCXVisionV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
"""
HyperCLOVAX-SEED Vision-Language Model (V2 architecture).
"""HyperCLOVAX-SEED Vision-Language Model (V2 architecture).
Supports:
- HyperCLOVAX-SEED-Think-32B: Vision + Text
@@ -45,8 +45,7 @@ from .vision import is_vit_use_data_parallel, run_dp_sharded_vision_model
class Idefics2VisionEmbeddings(nn.Module):
"""
This is a modified version of `siglip.modelign_siglip.SiglipVisionEmbeddings
"""This is a modified version of `siglip.modelign_siglip.SiglipVisionEmbeddings
` to enable images of variable
resolution.
@@ -153,7 +152,7 @@ class Idefics2VisionEmbeddings(nn.Module):
class Idefics2VisionAttention(nn.Module):
"""Multi-headed attention from 'Attention Is All You Need' paper"""
"""Multi-headed attention from 'Attention Is All You Need' paper."""
def __init__(
self,
@@ -308,10 +307,9 @@ class Idefics2EncoderLayer(nn.Module):
hidden_states: torch.Tensor,
attention_mask: torch.Tensor | None = None,
) -> torch.Tensor:
"""
Args:
hidden_states (`torch.FloatTensor`):
Input to the layer of shape `(batch, seq_len, embed_dim)`.
"""Args:
hidden_states (`torch.FloatTensor`):
Input to the layer of shape `(batch, seq_len, embed_dim)`.
"""
residual = hidden_states
@@ -326,13 +324,13 @@ class Idefics2EncoderLayer(nn.Module):
class Idefics2Encoder(nn.Module):
"""
Transformer encoder consisting of `config.num_hidden_layers` self attention
"""Transformer encoder consisting of `config.num_hidden_layers` self attention
layers. Each layer is a
[`Idefics2EncoderLayer`].
Args:
config: Idefics2Config
"""
def __init__(
@@ -368,14 +366,14 @@ class Idefics2Encoder(nn.Module):
inputs_embeds: torch.Tensor,
attention_mask: torch.Tensor | None = None,
) -> torch.Tensor:
r"""
Args:
inputs_embeds (torch.Tensor):
Optionally, instead of passing `input_ids` you can choose to
directly pass an embedded representation.
This is useful if you want more control over how to convert
`input_ids` indices into associated vectorsthan the model's
internal embedding lookup matrix.
r"""Args:
inputs_embeds (torch.Tensor):
Optionally, instead of passing `input_ids` you can choose to
directly pass an embedded representation.
This is useful if you want more control over how to convert
`input_ids` indices into associated vectorsthan the model's
internal embedding lookup matrix.
"""
hidden_states = inputs_embeds
for encoder_layer in self.layers:
+11 -15
View File
@@ -70,13 +70,12 @@ from .utils import AutoWeightsLoader, maybe_prefix
class Idefics3ImagePixelInputs(TensorSchema):
"""
Dimensions:
- bn: Batch size * number of images
- bnp: Batch size * number of images * number of patches
- c: Number of channels (3)
- h: Height
- w: Width
"""Dimensions:
- bn: Batch size * number of images
- bnp: Batch size * number of images * number of patches
- c: Number of channels (3)
- h: Height
- w: Width
"""
type: Literal["pixel_values"]
@@ -86,11 +85,10 @@ class Idefics3ImagePixelInputs(TensorSchema):
class Idefics3ImageEmbeddingInputs(TensorSchema):
"""
Dimensions:
- bn: Batch size * number of images
- f: Image feature size
- h: Hidden size (must match the hidden size of language model backbone)
"""Dimensions:
- bn: Batch size * number of images
- f: Image feature size
- h: Hidden size (must match the hidden size of language model backbone)
"""
type: Literal["image_embeds"]
@@ -829,9 +827,7 @@ class Idefics3ForConditionalGeneration(
return loader.load_weights(weights)
def get_mm_mapping(self) -> MultiModelKeys:
"""
Get the module prefix in multimodal models
"""
"""Get the module prefix in multimodal models."""
return MultiModelKeys.from_string_field(
language_model="model.text_model",
connector="model.connector",
+51 -56
View File
@@ -112,8 +112,7 @@ class StreamingTranscriptionPostProcessor:
def _require_is_multimodal(is_multimodal: Tensor | None) -> Tensor:
"""
A helper function to be used in the context of
"""A helper function to be used in the context of
[vllm.model_executor.models.interfaces.SupportsMultiModal.embed_input_ids][]
to provide a better error message.
"""
@@ -225,20 +224,18 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
"""
Get the placeholder text for the `i`th `modality` item in the prompt.
"""
"""Get the placeholder text for the `i`th `modality` item in the prompt."""
...
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
"""
Returns multimodal embeddings generated from multimodal kwargs
"""Returns multimodal embeddings generated from multimodal kwargs
to be merged with text embeddings.
Note:
The returned multimodal embeddings must be in the same order as
the appearances of their corresponding multimodal data item in the
input prompt.
"""
...
@@ -254,14 +251,14 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
)
def get_language_model(self) -> "VllmModel":
"""
Returns the underlying language model used for text generation.
"""Returns the underlying language model used for text generation.
This is typically the `torch.nn.Module` instance responsible for
processing the merged multimodal embeddings and producing hidden states
Returns:
torch.nn.Module: The core language model component.
"""
# Cached
assert isinstance(self, nn.Module)
@@ -302,8 +299,7 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
*,
targets: type[nn.Module] | tuple[type[nn.Module], ...] | None = None,
):
"""
Mark each child module that was assigned to this model during this context
"""Mark each child module that was assigned to this model during this context
as a language model component.
Language model components are automatically skipped in `--mm-encoder-only`
@@ -338,8 +334,7 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
*,
targets: type[nn.Module] | tuple[type[nn.Module], ...] | None = None,
):
"""
Mark each child module that was assigned to this model during this context
"""Mark each child module that was assigned to this model during this context
as a tower model component.
Tower model components are automatically skipped when `--limit-mm-per-prompt`
@@ -397,8 +392,7 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
language_targets: type[nn.Module] | tuple[type[nn.Module], ...],
tower_targets: dict[str, type[nn.Module] | tuple[type[nn.Module], ...]],
):
"""
Composite wrapper over `_mark_language_model` and
"""Composite wrapper over `_mark_language_model` and
`_mark_tower_model` by modality.
"""
with ExitStack() as stack:
@@ -438,8 +432,7 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
mm_kwargs: "MultiModalKwargsItem | None",
num_mm_embeds: int,
) -> tuple[int, int | None]:
"""
Return ``(tower_tokens, connector_tokens)`` for multimodal LoRA mappings.
"""Return ``(tower_tokens, connector_tokens)`` for multimodal LoRA mappings.
MM LoRA uses these counts to build adapter mappings for the tower and
connector forwards. Models with multiple modalities can override this
@@ -497,8 +490,7 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
*,
is_multimodal: Tensor | None = None,
) -> Tensor:
"""
Apply token embeddings to `input_ids`.
"""Apply token embeddings to `input_ids`.
If `multimodal_embeddings` is passed, scatter them into
`input_ids` according to the mask `is_multimodal`.
@@ -551,8 +543,7 @@ class SupportsMultiModalPruning(Protocol):
mrope_positions: torch.LongTensor,
num_computed_tokens: int,
) -> tuple[Sequence[torch.Tensor], Tensor, int]:
"""
Update part of input mrope positions (starting with
"""Update part of input mrope positions (starting with
num_computed_tokens index). Original mrope_positions are computed
for unpruned sequence and becomes incorrect once pruning occurs,
so once we prune media tokens we should reflect this in the
@@ -571,6 +562,7 @@ class SupportsMultiModalPruning(Protocol):
Returns:
Tuple of (multimodal_embeddings, mrope_positions,
mrope_position_delta).
"""
...
@@ -650,16 +642,12 @@ class SupportsScoreTemplate(Protocol):
@classmethod
def get_score_template(cls, query: str, document: str) -> str | None:
"""
Generate a full prompt by populating the score template with query and document content.
""" # noqa: E501
"""Generate a full prompt by populating the score template with query and document content.""" # noqa: E501
...
@classmethod
def post_process_tokens(cls, prompt: "TokensPrompt") -> None:
"""
Perform architecture-specific manipulations on the input tokens.
"""
"""Perform architecture-specific manipulations on the input tokens."""
...
@@ -790,8 +778,7 @@ class SupportsPP(Protocol):
*,
intermediate_tensors: "IntermediateTensors | None",
) -> "Tensor | IntermediateTensors | tuple[Tensor, list[Tensor]]":
"""
Accept [`IntermediateTensors`][vllm.sequence.IntermediateTensors] when
"""Accept [`IntermediateTensors`][vllm.sequence.IntermediateTensors] when
PP rank > 0.
Return [`IntermediateTensors`][vllm.sequence.IntermediateTensors] only
@@ -905,7 +892,8 @@ def has_inner_state(
@runtime_checkable
class IsAttentionFree(Protocol):
"""The interface required for all models like Mamba that lack attention,
but do have state whose size is constant wrt the number of tokens."""
but do have state whose size is constant wrt the number of tokens.
"""
is_attention_free: ClassVar[Literal[True]] = True
"""
@@ -933,7 +921,8 @@ def is_attention_free(
class IsHybrid(Protocol):
"""The interface required for all models like Jamba that have both
attention and mamba blocks, indicates that
hf_config has 'layers_block_type'"""
hf_config has 'layers_block_type'
"""
is_hybrid: ClassVar[Literal[True]] = True
"""
@@ -953,6 +942,7 @@ class IsHybrid(Protocol):
Returns:
Shapes for each state cache used by the model.
"""
...
@@ -966,6 +956,7 @@ class IsHybrid(Protocol):
(state, block_ids, cur_block_idx, num_accepted_tokens) and returns
a MambaCopySpec describing the memory-copy parameters for prefix
caching in align mode.
"""
...
@@ -994,9 +985,7 @@ def is_hybrid(
@runtime_checkable
class MixtureOfExperts(Protocol):
"""
Check if the model is a mixture of experts (MoE) model.
"""
"""Check if the model is a mixture of experts (MoE) model."""
expert_weights: MutableSequence[Sequence[Tensor]]
"""
@@ -1039,8 +1028,7 @@ class MixtureOfExperts(Protocol):
logical_to_physical_map: Tensor,
logical_replica_count: Tensor,
) -> None:
"""
Register the EPLB state in the MoE model.
"""Register the EPLB state in the MoE model.
Since these are views of the actual EPLB state, any changes made by
the EPLB algorithm are automatically reflected in the model's behavior
@@ -1054,6 +1042,7 @@ class MixtureOfExperts(Protocol):
expert_load_view: A view of the expert load metrics tensor.
logical_to_physical_map: Mapping from logical to physical experts.
logical_replica_count: Count of replicas for each logical expert.
"""
self.expert_weights = []
for layer_idx, layer in enumerate(self.moe_layers):
@@ -1086,8 +1075,8 @@ def get_mixture_of_experts_model(model: object) -> MixtureOfExperts | None:
Returns:
The MixtureOfExperts instance contained within the model, or None.
"""
"""
if is_mixture_of_experts(model):
return model
@@ -1349,7 +1338,8 @@ class SupportsTranscription(Protocol):
) -> "PromptType":
"""Get the prompt for the ASR model.
The model has control over the construction, as long as it
returns a valid PromptType."""
returns a valid PromptType.
"""
...
@classmethod
@@ -1359,8 +1349,7 @@ class SupportsTranscription(Protocol):
@classmethod
def validate_language(cls, language: str | None) -> str | None:
"""
Ensure the language specified in the transcription request
"""Ensure the language specified in the transcription request
is a valid ISO 639-1 language code. If the request language is
valid, but not natively supported by the model, trigger a
warning (but not an exception).
@@ -1396,8 +1385,7 @@ class SupportsTranscription(Protocol):
stt_config: "SpeechToTextConfig",
model_config: "ModelConfig",
) -> int | None:
"""
Map from audio duration to number of audio tokens produced by the ASR
"""Map from audio duration to number of audio tokens produced by the ASR
model, without running a forward pass.
This is used for estimating the amount of processing for this audio.
"""
@@ -1405,8 +1393,7 @@ class SupportsTranscription(Protocol):
@classmethod
def post_process_output(cls, text: str) -> str:
"""
Post-process the raw model output text.
"""Post-process the raw model output text.
Some ASR models output structured formats (e.g., language tags,
special tokens) that need to be stripped before returning to the user.
@@ -1416,6 +1403,7 @@ class SupportsTranscription(Protocol):
Returns:
Cleaned transcription text.
"""
return text
@@ -1432,8 +1420,7 @@ class SupportsTranscription(Protocol):
def get_streaming_post_processor_cls(
cls,
) -> type[StreamingTranscriptionPostProcessor]:
"""
Return a stateful post-processor class for streaming output deltas.
"""Return a stateful post-processor class for streaming output deltas.
Each instance receives the next decoded text delta and whether the
request output is final. It returns the cleaned delta that should be
@@ -1644,7 +1631,8 @@ class EagleModelMixin:
@runtime_checkable
class SupportsEagle(SupportsEagleBase, Protocol):
"""The interface required for models that support
EAGLE-1 and EAGLE-2 speculative decoding."""
EAGLE-1 and EAGLE-2 speculative decoding.
"""
supports_eagle: ClassVar[Literal[True]] = True
"""
@@ -1674,7 +1662,8 @@ def supports_eagle(
@runtime_checkable
class SupportsEagle3(SupportsEagleBase, Protocol):
"""The interface required for models that support
EAGLE-3 speculative decoding."""
EAGLE-3 speculative decoding.
"""
supports_eagle3: ClassVar[Literal[True]] = True
"""
@@ -1687,12 +1676,12 @@ class SupportsEagle3(SupportsEagleBase, Protocol):
"""
def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
"""
Set which layers should output auxiliary hidden states for EAGLE-3.
"""Set which layers should output auxiliary hidden states for EAGLE-3.
Args:
layers: Tuple of layer indices that should output auxiliary
hidden states.
"""
parent_ref = self
if hasattr(self, "get_language_model"):
@@ -1710,14 +1699,14 @@ class SupportsEagle3(SupportsEagleBase, Protocol):
holder._set_aux_hidden_state_layers(layers)
def get_eagle3_default_aux_hidden_state_layers(self) -> tuple[int, ...]:
"""
Get the default layer indices that should output auxiliary hidden states
"""Get the default layer indices that should output auxiliary hidden states
for EAGLE-3 for this model. Models can override this method to provide
different default layers based on their architecture, but it is encouraged
to instead include the layer specification in the model's config if possible.
Returns:
Tuple of layer indices for auxiliary hidden state outputs.
"""
parent_ref = self
if hasattr(self, "get_language_model"):
@@ -1765,8 +1754,7 @@ class SupportsMRoPE(Protocol):
input_tokens: list[int],
mm_features: list["MultiModalFeatureSpec"],
) -> tuple[torch.Tensor, int]:
"""
Get M-RoPE input positions and delta value for this specific model.
"""Get M-RoPE input positions and delta value for this specific model.
This method should be implemented by each model that supports M-RoPE
to provide model-specific logic for computing input positions.
@@ -1779,6 +1767,7 @@ class SupportsMRoPE(Protocol):
llm_positions: Tensor of shape `[num_dims, num_tokens]`, one row
per M-RoPE position channel (e.g. T/H/W).
mrope_position_delta: Delta for position calculations.
"""
...
@@ -1886,8 +1875,7 @@ class SupportsEncoderCudaGraph(Protocol):
clone: bool = False,
batch_mm_kwargs: dict[str, Any] | None = None,
) -> None:
"""
Post-process encoder output, directly call scatter_output_slices by default.
"""Post-process encoder output, directly call scatter_output_slices by default.
By default, delegates directly to scatter_output_slices.
Override this for models that require additional processing on the raw
@@ -1913,10 +1901,17 @@ class SupportsEncoderCudaGraph(Protocol):
"""Create dummy inputs and buffers for CUDA graph capture.
Args:
token_budget: Token budget the capture is sized for.
max_batch_size: Maximum number of items in a captured batch.
max_frames_per_batch: Maximum number of frames in a captured batch.
device: Device the dummy inputs and buffers are created on.
dtype: Dtype of the dummy inputs and buffers.
path: Configured encoder path.
axis_keys: The resolved capture-axis keys (one per axis of
``EncoderCudaGraphConfig.capture_axes``) this capture is for.
None or empty when no capture axes are configured; models
without capture axes ignore it.
"""
...
+1 -1
View File
@@ -143,7 +143,7 @@ class InternVisionPatchModel(nn.Module):
class InternParallelAttention(nn.Module):
"""Multi-headed attention from 'Attention Is All You Need' paper"""
"""Multi-headed attention from 'Attention Is All You Need' paper."""
def __init__(
self,
+21 -27
View File
@@ -89,13 +89,12 @@ class InternS1MultiModalProjector(nn.Module):
class InternS1ImagePixelInputs(TensorSchema):
"""
Dimensions:
- bnp: Batch size * number of images * (1 + num_patches)
- c: Number of channels (3)
- h: Height
- w: Width
- bn: Batch size * number of images
"""Dimensions:
- bnp: Batch size * number of images * (1 + num_patches)
- c: Number of channels (3)
- h: Height
- w: Width
- bn: Batch size * number of images
"""
type: Literal["pixel_values"] = "pixel_values"
@@ -104,11 +103,10 @@ class InternS1ImagePixelInputs(TensorSchema):
class InternS1ImageEmbeddingInputs(TensorSchema):
"""
Dimensions:
- ni: Number of images
- tifs: Total image feature size
- hs: Hidden size (must match language model backbone)
"""Dimensions:
- ni: Number of images
- tifs: Total image feature size
- hs: Hidden size (must match language model backbone)
"""
type: Literal["image_embeds"] = "image_embeds"
@@ -119,13 +117,12 @@ InternS1ImageInputs: TypeAlias = InternS1ImagePixelInputs | InternS1ImageEmbeddi
class InternS1VideoPixelInputs(TensorSchema):
"""
Dimensions:
- bnv: Batch size * number of videos * number of frames
- bn: Batch size * number of images
- c: Number of channels (3)
- h: Height
- w: Width
"""Dimensions:
- bnv: Batch size * number of videos * number of frames
- bn: Batch size * number of images
- c: Number of channels (3)
- h: Height
- w: Width
"""
type: Literal["pixel_values_videos"] = "pixel_values_videos"
@@ -134,11 +131,10 @@ class InternS1VideoPixelInputs(TensorSchema):
class InternS1VideoEmbeddingInputs(TensorSchema):
"""
Dimensions:
- nv: Number of videos
- tvfs: Total video feature size
- hs: Hidden size (must match language model backbone)
"""Dimensions:
- nv: Number of videos
- tvfs: Total video feature size
- hs: Hidden size (must match language model backbone)
"""
type: Literal["video_embeds"] = "video_embeds"
@@ -862,9 +858,7 @@ class InternS1ForConditionalGeneration(
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def get_mm_mapping(self) -> MultiModelKeys:
"""
Get the module prefix in multimodal models
"""
"""Get the module prefix in multimodal models."""
return MultiModelKeys.from_string_field(
language_model="language_model",
connector="multi_modal_projector",
+1 -1
View File
@@ -621,7 +621,7 @@ class InternS1ProForConditionalGeneration(
return mapper
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
"""load weights"""
"""Load weights"""
orig_to_new_prefix: dict[str, str | None] = {
"model.visual.": "visual.",
"lm_head.": "language_model.lm_head.",
+3 -6
View File
@@ -91,15 +91,13 @@ class InternS1VisionEmbeddings(nn.Module):
def interpolate_pos_encoding(
self, embeddings: torch.Tensor, height: int, width: int
) -> torch.Tensor:
"""
This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
"""This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
images. This method is also adapted to support torch.jit tracing.
Adapted from:
- https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
- https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
""" # noqa: E501
num_patches = embeddings.shape[1] - 1
num_positions = self.position_embeddings.shape[1] - 1
@@ -164,7 +162,7 @@ class InternS1VisionEmbeddings(nn.Module):
class InternSdpaAttention(nn.Module):
"""Multi-headed attention from 'Attention Is All You Need' paper"""
"""Multi-headed attention from 'Attention Is All You Need' paper."""
def __init__(
self,
@@ -225,8 +223,7 @@ class InternSdpaAttention(nn.Module):
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
"""x shape: (B, N, C)"""
"""X shape: (B, N, C)."""
q = self.q_proj(x)
k = self.k_proj(x)
v = self.v_proj(x)
+24 -30
View File
@@ -70,13 +70,12 @@ from .utils import (
class InternVLImagePixelInputs(TensorSchema):
"""
Dimensions:
- bn: Batch size * number of images
- bnp: Batch size * number of images * (1 + num_patches)
- c: Number of channels (3)
- h: Height of each image patch
- w: Width of each image patch
"""Dimensions:
- bn: Batch size * number of images
- bnp: Batch size * number of images * (1 + num_patches)
- c: Number of channels (3)
- h: Height of each image patch
- w: Width of each image patch
"""
type: Literal["pixel_values"]
@@ -85,11 +84,10 @@ class InternVLImagePixelInputs(TensorSchema):
class InternVLImageEmbeddingInputs(TensorSchema):
"""
Dimensions:
- n: Number of images
- f: Total image feature size
- h: Hidden size (must match the hidden size of language model backbone)
"""Dimensions:
- n: Number of images
- f: Total image feature size
- h: Hidden size (must match the hidden size of language model backbone)
"""
type: Literal["image_embeds"]
@@ -100,13 +98,12 @@ InternVLImageInputs: TypeAlias = InternVLImagePixelInputs | InternVLImageEmbeddi
class InternVLVideoPixelInputs(TensorSchema):
"""
Dimensions:
- bvf: Batch size * number of videos * num_frames
- bn: Batch size * number of images
- c: Number of channels (3)
- h: Height of each video frame
- w: Width of each video frame
"""Dimensions:
- bvf: Batch size * number of videos * num_frames
- bn: Batch size * number of images
- c: Number of channels (3)
- h: Height of each video frame
- w: Width of each video frame
"""
type: Literal["pixel_values_videos"]
@@ -115,11 +112,10 @@ class InternVLVideoPixelInputs(TensorSchema):
class InternVLVideoEmbeddingInputs(TensorSchema):
"""
Dimensions:
- n: Number of videos
- f: Total video feature size
- h: Hidden size (must match the hidden size of language model backbone)
"""Dimensions:
- n: Number of videos
- f: Total video feature size
- h: Hidden size (must match the hidden size of language model backbone)
"""
type: Literal["video_embeds"]
@@ -327,7 +323,7 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
class InternVLProcessingInfo(BaseInternVLProcessingInfo):
"""InternVL ProcessingInfo extended for video processing"""
"""InternVL ProcessingInfo extended for video processing."""
def get_image_processor(self, **kwargs):
config = self.get_hf_config()
@@ -418,7 +414,7 @@ class InternVLProcessingInfo(BaseInternVLProcessingInfo):
class InternVLDummyInputsBuilder(
BaseInternVLDummyInputsBuilder[InternVLProcessingInfo]
):
"""InternVL DummyInputsBuilder extended for video support"""
"""InternVL DummyInputsBuilder extended for video support."""
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
num_videos = mm_counts.get("video", 0)
@@ -460,7 +456,7 @@ class InternVLDummyInputsBuilder(
class InternVLMultiModalProcessor(
BaseInternVLMultiModalProcessor[InternVLProcessingInfo]
):
"""InternVL MultiModalProcessor extended for video support"""
"""InternVL MultiModalProcessor extended for video support."""
def _postprocess_hf_mm_data(
self,
@@ -906,9 +902,7 @@ class InternVLChatModel(
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def get_mm_mapping(self) -> MultiModelKeys:
"""
Get the module prefix in multimodal models
"""
"""Get the module prefix in multimodal models."""
return MultiModelKeys.from_string_field(
language_model="language_model",
connector="mlp1",
@@ -321,6 +321,7 @@ class LoopGateProjection(nn.Module):
Returns:
gate: [num_tokens, num_heads * head_dim] (flattened format matching q shape)
"""
num_heads, num_tokens, head_dim = query.shape
+10 -14
View File
@@ -177,23 +177,23 @@ def create_pixel_shuffle_index_map(
scale_factor: int = 1,
device: torch.device | None = None,
) -> torch.Tensor:
"""
Build a gather-index map that tells us, for every *output* token after
"""Build a gather-index map that tells us, for every *output* token after
pixel-shuffle, which `scale_factor**2` *input* tokens are being merged.
Args
Args:
----
seq_sizes : (num_images,) - #patches in each image (row-major order)
token_grids : (num_images,2) - (height, width) for every image
scale_factor : spatial down-scale factor (≥2)
device : (optional) overrides `seq_sizes.device`
Returns
Returns:
-------
gather_idx : (new_total_seq_len, scale_factor**2) int64 tensor.
gather_idx[i, j] is the *flat* index into the *original*
packed sequence for the j-th sub-patch that forms the
i-th output token.
"""
if device is None:
device = seq_sizes.device
@@ -266,6 +266,7 @@ def pixel_shuffle_varlen(
Raises:
ValueError: If more than one batch item is provided.
"""
keep_batch_dim = x.dim() == 3
if keep_batch_dim:
@@ -403,8 +404,7 @@ class IsaacDummyInputsBuilder(BaseDummyInputsBuilder[IsaacProcessingInfo]):
class IsaacImagePixelInputs(TensorSchema):
"""
Schema for validating Isaac image inputs.
"""Schema for validating Isaac image inputs.
Dimensions:
- np: Number of patches
@@ -687,12 +687,10 @@ class Siglip2VisionTransformer(nn.Module):
self,
packed_seq_patches: tuple[torch.Tensor, torch.Tensor],
) -> torch.Tensor:
r"""
spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`):
Tensor containing the spatial dimensions (height, width)
of the input images.
r"""spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`):
Tensor containing the spatial dimensions (height, width)
of the input images.
"""
seq_patches, token_grids = packed_seq_patches
seq_sizes = torch.prod(token_grids, dim=-1)
@@ -1015,9 +1013,7 @@ class IsaacForConditionalGeneration(
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def get_mm_mapping(self) -> MultiModelKeys:
"""
Get the module prefix in multimodal models
"""
"""Get the module prefix in multimodal models."""
return MultiModelKeys.from_string_field(
language_model="language_model",
connector="vision_embedding.linear_fc2", # The final linear layer
+4 -5
View File
@@ -48,11 +48,10 @@ logger = init_logger(__name__)
class KananaVImagePixelInputs(TensorSchema):
"""
Dimensions:
- np: The total number of patches over all images in the batch
- cps: Number of channels * patch_size * patch_size
- ni: Number of images
"""Dimensions:
- np: The total number of patches over all images in the batch
- cps: Number of channels * patch_size * patch_size
- ni: Number of images
"""
type: Literal["pixel_values"]
+30 -31
View File
@@ -125,13 +125,12 @@ def smart_resize(
class KeyeImagePixelInputs(TensorSchema):
"""
Dimensions:
- bnp: Batch size * Number of patches
- c: Number of channels
- ps: Patch size
- ni: Number of images
- g: Grid dimensions (3 for t, h, w)
"""Dimensions:
- bnp: Batch size * Number of patches
- c: Number of channels
- ps: Patch size
- ni: Number of images
- g: Grid dimensions (3 for t, h, w)
"""
type: Literal["pixel_values"]
@@ -142,13 +141,12 @@ class KeyeImagePixelInputs(TensorSchema):
class KeyeImageEmbeddingInputs(TensorSchema):
"""
Dimensions:
- nf: Number of image features
- hs: Hidden size (must match the hidden size of language model
backbone)
- ni: Number of images
- g: Grid dimensions (3 for t, h, w)
"""Dimensions:
- nf: Number of image features
- hs: Hidden size (must match the hidden size of language model
backbone)
- ni: Number of images
- g: Grid dimensions (3 for t, h, w)
"""
type: Literal["image_embeds"]
@@ -160,13 +158,12 @@ KeyeImageInputs: TypeAlias = KeyeImagePixelInputs | KeyeImageEmbeddingInputs
class KeyeVideoPixelInputs(TensorSchema):
"""
Dimensions:
- bnp: Batch size * Number of patches
- c: Number of channels
- ps: Patch size
- ni: Number of images
- g: Grid dimensions (3 for t, h, w)
"""Dimensions:
- bnp: Batch size * Number of patches
- c: Number of channels
- ps: Patch size
- ni: Number of images
- g: Grid dimensions (3 for t, h, w)
"""
type: Literal["pixel_values_videos"]
@@ -177,13 +174,12 @@ class KeyeVideoPixelInputs(TensorSchema):
class KeyeVideoEmbeddingInputs(TensorSchema):
"""
Dimensions:
- nf: Number of video features
- hs: Hidden size (must match the hidden size of language model
backbone)
- nv: Number of videos
- g: Grid dimensions (3 for t, h, w)
"""Dimensions:
- nf: Number of video features
- hs: Hidden size (must match the hidden size of language model
backbone)
- nv: Number of videos
- g: Grid dimensions (3 for t, h, w)
"""
type: Literal["video_embeds"]
@@ -349,7 +345,8 @@ def apply_rotary_pos_emb_flashatt(
class KeyeSiglipAttention(nn.Module):
"""Multi-headed attention from 'Attention Is All You
Need' paper."""
Need' paper.
"""
def __init__(
self,
@@ -1465,6 +1462,9 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):
otherwise it will be `(seq_len,)`.
intermediate_tensors: Intermediate tensors from prior forward pass.
inputs_embeds: Optional tensor of input embeddings.
**kwargs: Multimodal inputs for this batch, forwarded to the
multimodal embedding path.
"""
if intermediate_tensors is not None:
inputs_embeds = None
@@ -1588,8 +1588,7 @@ class KeyeForConditionalGeneration(
def _split_video_grid_thw(
grid_thw: torch.Tensor | list[list[int]] | list[int],
) -> list[list[int]]:
"""
Split video grid_thw along the t dimension into per-frame rows.
"""Split video grid_thw along the t dimension into per-frame rows.
This preserves Keye's current M-RoPE behavior, where a video is emitted
as consecutive frame-level multimodal blocks rather than a single block
+32 -37
View File
@@ -51,8 +51,7 @@ logger = init_logger(__name__)
def split_thw(grid_thw: torch.Tensor) -> torch.Tensor:
"""
Split grid_thw in t dimension.
"""Split grid_thw in t dimension.
Args:
grid_thw: [N, 3] tensor of [t, h, w]
@@ -66,6 +65,7 @@ def split_thw(grid_thw: torch.Tensor) -> torch.Tensor:
tensor([[1, 3, 4],
[1, 3, 4],
[1, 5, 6]])
"""
t = grid_thw[:, 0]
h_w = grid_thw[:, 1:]
@@ -76,8 +76,7 @@ def split_thw(grid_thw: torch.Tensor) -> torch.Tensor:
def get_num_patches(
grid_thw: torch.Tensor, num_frames: list[int] | torch.Tensor
) -> list[int]:
"""
Return num_patches per video.
"""Return num_patches per video.
Args:
grid_thw: Tensor with shape [N, 3] containing temporal, height, width
@@ -101,8 +100,8 @@ def get_num_patches(
>>> get_num_patches(grid_thw, num_frames)
tensor([16, 1]) # Total patches for first video: 8+8=16,
second video: 1.
"""
"""
assert len(grid_thw.shape) == 2
if isinstance(num_frames, torch.Tensor):
num_frames = num_frames.clone().tolist()
@@ -121,13 +120,12 @@ def get_num_patches(
class KeyeVL1_5ImagePixelInputs(TensorSchema):
"""
Dimensions:
- bnp: Batch size * Number of patches
- c: Number of channels
- ps: Patch size
- ni: Number of images
- g: Grid dimensions (3 for t, h, w)
"""Dimensions:
- bnp: Batch size * Number of patches
- c: Number of channels
- ps: Patch size
- ni: Number of images
- g: Grid dimensions (3 for t, h, w)
"""
type: Literal["pixel_values"]
@@ -140,13 +138,12 @@ class KeyeVL1_5ImagePixelInputs(TensorSchema):
class KeyeVL1_5ImageEmbeddingInputs(TensorSchema):
"""
Dimensions:
- nf: Number of image features
- hs: Hidden size (must match the hidden size of language model
backbone)
- ni: Number of images
- g: Grid dimensions (3 for t, h, w)
"""Dimensions:
- nf: Number of image features
- hs: Hidden size (must match the hidden size of language model
backbone)
- ni: Number of images
- g: Grid dimensions (3 for t, h, w)
"""
type: Literal["image_embeds"]
@@ -160,13 +157,12 @@ KeyeVL1_5ImageInputs: TypeAlias = (
class KeyeVL1_5VideoPixelInputs(TensorSchema):
"""
Dimensions:
- bnp: Batch size * Number of patches
- c: Number of channels
- ps: Patch size
- ni: Number of images
- g: Grid dimensions (3 for t, h, w)
"""Dimensions:
- bnp: Batch size * Number of patches
- c: Number of channels
- ps: Patch size
- ni: Number of images
- g: Grid dimensions (3 for t, h, w)
"""
type: Literal["pixel_values_videos"]
@@ -179,13 +175,12 @@ class KeyeVL1_5VideoPixelInputs(TensorSchema):
class KeyeVL1_5VideoEmbeddingInputs(TensorSchema):
"""
Dimensions:
- nf: Number of video features
- hs: Hidden size (must match the hidden size of language model
backbone)
- nv: Number of videos
- g: Grid dimensions (3 for t, h, w)
"""Dimensions:
- nf: Number of video features
- hs: Hidden size (must match the hidden size of language model
backbone)
- nv: Number of videos
- g: Grid dimensions (3 for t, h, w)
"""
type: Literal["video_embeds"]
@@ -437,10 +432,10 @@ class KeyeVL1_5MultiModalProcessor(BaseMultiModalProcessor[KeyeVL1_5ProcessingIn
cu_seqlens = torch.cumsum(torch.tensor([0] + num_frames.tolist()), dim=-1)
def get_replacement_keye(item_idx: int, modality: str):
"""
Args:
item_idx(int): The item index of modality to replace
modality(str): The modality
"""Args:
item_idx(int): The item index of modality to replace
modality(str): The modality
"""
if modality == "image":
out_item = out_mm_kwargs[modality][item_idx]
+2 -4
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Kimi-K2.5 Model Implementation for vLLM.
"""Kimi-K2.5 Model Implementation for vLLM.
Kimi-K2.5 extends Kimi-K2 with vision support.
"""
@@ -90,8 +89,7 @@ class MaxImageTokenMeta:
class KimiK25MediaPixelInputs(TensorSchema):
"""
Media input schema for K2-VL model.
"""Media input schema for K2-VL model.
Dimensions:
- np: Number of patches (flattened from all media items)
+9 -6
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Vision tower implementation for Kimi-K2.5 model.
"""Vision tower implementation for Kimi-K2.5 model.
This module provides the vision encoder components for Kimi-K2.5,
including 3D patch embedding, RoPE position embedding, and
@@ -284,12 +283,12 @@ class Rope2DPosEmbRepeated(nn.Module):
def get_freqs_cis(
self, grid_thws: torch.Tensor | list[list[int]], device: torch.device
) -> torch.Tensor:
"""
Args:
"""Args:
grid_thws (torch.Tensor): grid time, height and width
Returns:
freqs_cis: tensor of shape (sum(t * height * width), dim//2)
"""
if not hasattr(self, "freqs_cis"):
self.register_buffer(
@@ -449,6 +448,10 @@ class MoonViTEncoderLayer(nn.Module):
Args:
x (torch.Tensor): (seqlen, hidden_dim)
cu_seqlens (torch.Tensor): cumulative sequence lengths
rope_freqs_cis (torch.Tensor): rotary embedding frequencies
max_seqlen (torch.Tensor | None): longest sequence in the batch
sequence_lengths (torch.Tensor | None): per-sequence lengths
"""
seq_length = x.size(0)
xqkv, _ = self.wqkv(x)
@@ -753,13 +756,13 @@ class MoonViT3dPretrainedModel(nn.Module):
*,
encoder_metadata: dict[str, torch.Tensor | None] | None = None,
) -> torch.Tensor:
"""
Args:
"""Args:
pixel_values (torch.Tensor): The input pixel values.
grid_thws (torch.Tensor): Temporal, height and width.
Returns:
torch.Tensor: The output tokens.
"""
if encoder_metadata is not None and "pos_embeds" in encoder_metadata:
hidden_states = self.patch_embed(
+5 -6
View File
@@ -135,12 +135,11 @@ class KimiVLMultiModalProjector(nn.Module):
class KimiVLImagePixelInputs(TensorSchema):
"""
Dimensions:
- nc: Number of channels
- np: Number of patches
- ps: Patch size
- ni: Number of images
"""Dimensions:
- nc: Number of channels
- np: Number of patches
- ps: Patch size
- ni: Number of images
"""
type: Literal["pixel_values"] = "pixel_values"
+1
View File
@@ -440,6 +440,7 @@ class Lfm2ForCausalLM(
Returns:
Tuple containing:
- conv_state_shape: Shape for convolutional state cache
"""
parallel_config = vllm_config.parallel_config
hf_config = vllm_config.model_config.hf_config
+1
View File
@@ -556,6 +556,7 @@ class Lfm2MoeForCausalLM(
Returns:
Tuple containing:
- conv_state_shape: Shape for convolutional state cache
"""
parallel_config = vllm_config.parallel_config
hf_config = vllm_config.model_config.hf_config
+18 -11
View File
@@ -1,7 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Implementation of Siglip2VisionModel intended to be only used
within a vision language model."""
within a vision language model.
"""
from collections.abc import Iterable
@@ -61,6 +62,7 @@ class Siglip2VisionEmbeddings(nn.Module):
Returns:
(1, total_tokens, embed_dim) packed embeddings.
"""
assert spatial_shapes.device.type == "cpu", (
"Expected `spatial_shapes` on CPU to avoid device-to-host sync in "
@@ -113,6 +115,7 @@ class Siglip2VisionEmbeddings(nn.Module):
Returns:
(total_tokens, embed_dim) packed positional embeddings, concatenated
in the same order as `lengths_list`.
"""
assert spatial_shapes.device.type == "cpu"
@@ -155,7 +158,7 @@ class Siglip2VisionEmbeddings(nn.Module):
class Siglip2Attention(nn.Module):
"""Multi-headed attention from 'Attention Is All You Need' paper"""
"""Multi-headed attention from 'Attention Is All You Need' paper."""
def __init__(
self,
@@ -304,11 +307,11 @@ class Siglip2EncoderLayer(nn.Module):
cu_seqlens: torch.Tensor,
max_seqlen: int | torch.Tensor,
) -> torch.Tensor:
"""
Args:
hidden_states: Input tensor of shape (batch, seq_len, embed_dim).
cu_seqlens: Cumulative sequence lengths tensor.
max_seqlen: Maximum sequence length.
"""Args:
hidden_states: Input tensor of shape (batch, seq_len, embed_dim).
cu_seqlens: Cumulative sequence lengths tensor.
max_seqlen: Maximum sequence length.
"""
residual = hidden_states
@@ -328,12 +331,12 @@ class Siglip2EncoderLayer(nn.Module):
class Siglip2Encoder(nn.Module):
"""
Transformer encoder consisting of `config.num_hidden_layers`
"""Transformer encoder consisting of `config.num_hidden_layers`
self attention layers. Each layer is a [`Siglip2EncoderLayer`].
Args:
config: PretrainedConfig
"""
def __init__(
@@ -430,8 +433,7 @@ class Siglip2VisionTransformer(nn.Module):
max_seqlen: torch.Tensor,
select_layers: list[int] | None = None,
) -> torch.Tensor:
r"""
spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`):
r"""spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`):
Tensor containing the spatial dimensions (height, width)
of the input images.
select_layers (`list[int]` or `None`, defaults to `None`):
@@ -500,10 +502,15 @@ class Siglip2Model(torch.nn.Module):
"""Forward pass through the vision model.
Args:
pixel_values_packed: Packed pixel values for all images.
spatial_shapes: Per-image spatial dimensions.
cu_seqlens: Cumulative sequence lengths of the packed images.
max_seqlen: Longest image sequence in the batch.
select_layers: Layer indices to select hidden states from.
Supports negative indices (e.g., [-2] for second-to-last).
If None, returns the last layer output with post_layernorm.
Multiple layers can be selected and will be concatenated.
"""
return self.vision_model(
pixel_values_packed=pixel_values_packed,
+8 -9
View File
@@ -81,12 +81,11 @@ def _pad_cumulative_seqlens_buffer(
class Lfm2VLImagePixelInputs(TensorSchema):
"""
Dimensions:
- b: Number of images in the prompt
- bn: Batch size * number of images
- d: Number of dimensions
- fd: Number of features per dimension
"""Dimensions:
- b: Number of images in the prompt
- bn: Batch size * number of images
- d: Number of dimensions
- fd: Number of features per dimension
"""
type: Literal["pixel_values"] = "pixel_values"
@@ -538,6 +537,7 @@ class Lfm2VLMultiModalProjector(nn.Module):
Returns:
projected_packed: (total_projected_tokens, text_hidden_size)
"""
assert spatial_shapes.device.type == "cpu", (
"Expected `spatial_shapes` on CPU to avoid device-to-host sync in "
@@ -667,6 +667,7 @@ class Lfm2VLForConditionalGeneration(
Returns:
Tuple containing:
- conv_state_shape: Shape for convolutional state cache
"""
parallel_config = vllm_config.parallel_config
hf_language_config = vllm_config.model_config.hf_config.text_config
@@ -1276,9 +1277,7 @@ class Lfm2VLForConditionalGeneration(
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def get_mm_mapping(self) -> MultiModelKeys:
"""
Get the module prefix in multimodal models
"""
"""Get the module prefix in multimodal models."""
return MultiModelKeys.from_string_field(
language_model="language_model",
connector="multi_modal_projector",
+2 -3
View File
@@ -406,8 +406,7 @@ class Llama4Model(LlamaModel):
expert_params_mapping: list[tuple[str, str, int, str]],
fused: bool = True,
) -> bool:
"""
Load MoE expert weights.
"""Load MoE expert weights.
Args:
name: The name of the weight to load.
@@ -427,8 +426,8 @@ class Llama4Model(LlamaModel):
Returns:
True if loaded_weight is one of MoE weights and the MoE expert
weights are loaded successfully, False otherwise.
"""
"""
# Whether the MoE expert weights are loaded successfully.
expert_param_loaded = False
+11 -12
View File
@@ -72,8 +72,7 @@ from .vision import get_num_selected_vision_tokens, get_vision_encoder_info
class LlavaImagePixelInputs(TensorSchema):
"""
Dimensions:
"""Dimensions:
- bn: Batch size * number of images
- c: Number of channels (3)
- h: Height
@@ -88,8 +87,7 @@ class LlavaImagePixelInputs(TensorSchema):
class PixtralHFImagePixelInputs(TensorSchema):
"""
Dimensions:
"""Dimensions:
- bn: Batch size * number of images
- c: Number of channels
- h: Height
@@ -107,11 +105,10 @@ class PixtralHFImagePixelInputs(TensorSchema):
class LlavaImageEmbeddingInputs(TensorSchema):
"""
Dimensions:
- bn: Batch size * number of images
- ifs: Image feature size
- hs: Hidden size (must match language model backbone)
"""Dimensions:
- bn: Batch size * number of images
- ifs: Image feature size
- hs: Hidden size (must match language model backbone)
"""
type: Literal["image_embeds"] = "image_embeds"
@@ -440,6 +437,7 @@ def _get_num_hidden_layers(hf_config: LlavaLikeConfig) -> int:
Args:
hf_config: Model config with vision feature layer(s).
"""
feature_layers = hf_config.vision_feature_layer
num_hidden_layers = hf_config.vision_config.num_hidden_layers
@@ -707,9 +705,12 @@ class LlavaForConditionalGeneration(
positions: Position indices for the input tokens.
intermediate_tensors: Intermediate tensors from prior forward pass.
inputs_embeds: Optional tensor of input embeddings.
**kwargs: Multimodal inputs for this batch, forwarded to the
multimodal embedding path.
Info:
[`LlavaImageInputs`][vllm.model_executor.models.llava.LlavaImageInputs]
"""
if intermediate_tensors is not None:
inputs_embeds = None
@@ -731,9 +732,7 @@ class LlavaForConditionalGeneration(
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def get_mm_mapping(self) -> MultiModelKeys:
"""
Get the module prefix in multimodal models
"""
"""Get the module prefix in multimodal models."""
return MultiModelKeys.from_string_field(
language_model="language_model",
connector="multi_modal_projector",
+9 -10
View File
@@ -47,8 +47,7 @@ from .vision import get_num_selected_vision_tokens, get_vision_encoder_info
class LlavaNextImagePixelInputs(TensorSchema):
"""
Dimensions:
"""Dimensions:
- bn: Batch size * number of images
- np: Number of patches + 1
- c: Number of channels (3)
@@ -70,11 +69,10 @@ class LlavaNextImagePixelInputs(TensorSchema):
class LlavaNextImageEmbeddingInputs(TensorSchema):
"""
Dimensions:
- bn: Batch size * number of images
- ifs: Image feature size
- hs: Hidden size (must match language model backbone)
"""Dimensions:
- bn: Batch size * number of images
- ifs: Image feature size
- hs: Hidden size (must match language model backbone)
"""
type: Literal["image_embeds"] = "image_embeds"
@@ -578,9 +576,12 @@ model_executor.models.llava_next.LlavaNextProcessingInfo.get_num_image_tokens].
positions: Position indices for the input tokens.
intermediate_tensors: Intermediate tensors from prior forward pass.
inputs_embeds: Optional tensor of input embeddings.
**kwargs: Multimodal inputs for this batch, forwarded to the
multimodal embedding path.
Info:
[`LlavaNextImageInputs`][vllm.model_executor.models.llava_next.LlavaNextImageInputs]
"""
if intermediate_tensors is not None:
inputs_embeds = None
@@ -601,9 +602,7 @@ model_executor.models.llava_next.LlavaNextProcessingInfo.get_num_image_tokens].
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def get_mm_mapping(self) -> MultiModelKeys:
"""
Get the module prefix in multimodal models
"""
"""Get the module prefix in multimodal models"""
return MultiModelKeys.from_string_field(
language_model="language_model",
connector="multi_modal_projector",
+11 -8
View File
@@ -56,8 +56,7 @@ from .vision import get_vision_encoder_info
class LlavaNextVideoPixelInputs(TensorSchema):
"""
Dimensions:
"""Dimensions:
- bn: Batch size * number of videos
- f: Number of frames
- c: Number of channels (3)
@@ -381,8 +380,7 @@ class LlavaNextVideoForConditionalGeneration(
def _parse_and_validate_video_input(
self, **kwargs: object
) -> LlavaNextVideoPixelInputs | None:
"""
A legal video input should have the following dimensions:
"""A legal video input should have the following dimensions:
{
"pixel_values_videos" :
list[b, Tensor(nb_frames, nb_channels, height, width)]
@@ -457,10 +455,17 @@ class LlavaNextVideoForConditionalGeneration(
**kwargs: object,
) -> torch.Tensor | IntermediateTensors:
"""Run forward pass for LlaVA-NeXT-Video.
Args:
input_ids: Flattened (concatenated) input_ids corresponding to a
batch.
pixel_values_videos: Pixels in each frames for each input videos.
positions: Flattened (concatenated) position ids corresponding to a
batch.
intermediate_tensors: Intermediate tensors from prior forward pass.
inputs_embeds: Optional tensor of input embeddings.
**kwargs: Multimodal inputs for this batch, forwarded to the
multimodal embedding path.
"""
if intermediate_tensors is not None:
inputs_embeds = None
@@ -486,9 +491,7 @@ class LlavaNextVideoForConditionalGeneration(
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def get_mm_mapping(self) -> MultiModelKeys:
"""
Get the module prefix in multimodal models
"""
"""Get the module prefix in multimodal models."""
return MultiModelKeys.from_string_field(
language_model="language_model",
connector="multi_modal_projector",
+30 -27
View File
@@ -53,17 +53,16 @@ _MAX_FRAMES_PER_VIDEO = 16
class LlavaOnevisionVideoPixelInputs(TensorSchema):
"""
Dimensions:
- bn: Batch size * number of videos
- f: Number of frames
- c: Number of channels (3)
- h: Height
- w: Width
"""Dimensions:
- bn: Batch size * number of videos
- f: Number of frames
- c: Number of channels (3)
- h: Height
- w: Width
Note that `f` may be different for each batch, and 'num_frames'
may be different for each video, in which case the data is passed as a
list instead of a batched tensor.
Note that `f` may be different for each batch, and 'num_frames'
may be different for each video, in which case the data is passed as a
list instead of a batched tensor.
"""
type: Literal["pixel_values_videos"] = "pixel_values_videos"
@@ -75,16 +74,15 @@ class LlavaOnevisionVideoPixelInputs(TensorSchema):
class LlavaOnevisionImagePixelInputs(TensorSchema):
"""
Dimensions:
- bn: Batch size * number of images
- np: Number of patches (1 + num_patches)
- c: Number of channels (3)
- h: Height
- w: Width
"""Dimensions:
- bn: Batch size * number of images
- np: Number of patches (1 + num_patches)
- c: Number of channels (3)
- h: Height
- w: Width
Note that `num_patches` may be different per batch and image,
in which case the data is passed as a list instead of a batched tensor.
Note that `num_patches` may be different per batch and image,
in which case the data is passed as a list instead of a batched tensor.
"""
type: Literal["pixel_values"] = "pixel_values"
@@ -98,11 +96,10 @@ class LlavaOnevisionImagePixelInputs(TensorSchema):
class LlavaOnevisionImageEmbeddingInputs(TensorSchema):
"""
Dimensions:
- bn: Batch size * number of images
- ifs: Image feature size
- hs: Hidden size (must match language model backbone)
"""Dimensions:
- bn: Batch size * number of images
- ifs: Image feature size
- hs: Hidden size (must match language model backbone)
"""
type: Literal["image_embeds"] = "image_embeds"
@@ -557,8 +554,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
def _parse_and_validate_video_input(
self, **kwargs: object
) -> LlavaOnevisionVideoPixelInputs | None:
"""
A legal video input should have the following dimensions:
"""A legal video input should have the following dimensions:
{
"pixel_values_videos" :
list[b, Tensor(nb_frames, nb_channels, height, width)]
@@ -891,10 +887,17 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
**kwargs: object,
) -> torch.Tensor | IntermediateTensors:
"""Run forward pass for LlaVA-Onevision.
Args:
input_ids: Flattened (concatenated) input_ids corresponding to a
batch.
pixel_values_videos: Pixels in each frames for each input videos.
positions: Flattened (concatenated) position ids corresponding to a
batch.
intermediate_tensors: Intermediate tensors from prior forward pass.
inputs_embeds: Optional tensor of input embeddings.
**kwargs: Multimodal inputs for this batch, forwarded to the
multimodal embedding path.
"""
if intermediate_tensors is not None:
inputs_embeds = None
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Inference-only LLaVA-OneVision-2 (OV2) model for vLLM.
"""Inference-only LLaVA-OneVision-2 (OV2) model for vLLM.
Architecture notes:
@@ -313,8 +313,7 @@ class LongCatFlashMTP(nn.Module):
def _rewrite_spec_layer_name(
self, spec_layer: int, name: str, new_to_old_names_mapping: dict
) -> str:
"""
Rewrite the weight name to match the format of the original model.
"""Rewrite the weight name to match the format of the original model.
Add .mtp_block for modules in transformer layer block for spec layer
and rename shared layer weights to be top level.
"""
@@ -156,6 +156,7 @@ class NgramEmbedding(nn.Module):
oe_ids: ``[num_tokens, num_embedders]`` global (offset) n-gram ids,
as produced by the ``ngram_compute_n_gram_ids`` kernel.
Returns: ``[num_tokens, hidden]``.
"""
word = self.word_embeddings(input_ids) # [N, H]
flat = oe_ids.permute(1, 0).contiguous() # [num_embedders, N]
+1
View File
@@ -197,6 +197,7 @@ class Mamba2ForCausalLM(
Tuple containing:
- conv_state_shape: Shape for convolutional state cache
- temporal_state_shape: Shape for state space model cache
"""
parallel_config = vllm_config.parallel_config
hf_config = vllm_config.model_config.hf_config
+2 -1
View File
@@ -51,7 +51,8 @@ class Medusa(nn.Module):
truncated_vocab_size < vocab_size. To use this technique, one has to find
the top-k most frequent tokens in target dataset and add that as a tensor
in the draft checkpoint (using key token_map). Also, the draft config
needs to have truncated_vocab_size (=k) as an attribute."""
needs to have truncated_vocab_size (=k) as an attribute.
"""
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
speculative_config = vllm_config.speculative_config
+4 -8
View File
@@ -27,8 +27,7 @@ from .utils import PPMissingLayer, extract_layer_index, maybe_prefix
class MellumAttention(Qwen3MoeAttention):
"""
Differences from `Qwen3MoeAttention`:
"""Differences from `Qwen3MoeAttention`:
- Supports `per_layer_sliding_window` for `Attention`.
"""
@@ -122,8 +121,7 @@ class MellumAttention(Qwen3MoeAttention):
class MellumDecoderLayer(Qwen3MoeDecoderLayer):
"""
Differences from `Qwen3MoeDecoderLayer`:
"""Differences from `Qwen3MoeDecoderLayer`:
- Supports interleaved SWA and per-layer RoPE scaling.
"""
@@ -185,8 +183,7 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer):
@support_torch_compile
class MellumModel(Qwen3MoeModel):
"""
Differences from `Qwen3MoeModel`:
"""Differences from `Qwen3MoeModel`:
- Uses `MellumDecoderLayer`.
"""
@@ -204,8 +201,7 @@ class MellumModel(Qwen3MoeModel):
class MellumForCausalLM(Qwen3MoeForCausalLM):
"""
Differences from `Qwen3MoeForCausalLM`:
"""Differences from `Qwen3MoeForCausalLM`:
- Uses `MellumModel`.
"""
+3 -5
View File
@@ -511,11 +511,9 @@ class AudioProjectorSubsample(nn.Module):
# === Audio Inputs === #
class MiDashengLMAudioInputs(TensorSchema):
"""
Dimensions:
- bn: Batch size * number of audios
- p: Number of sampling points
"""Dimensions:
- bn: Batch size * number of audios
- p: Number of sampling points
"""
input_values: Annotated[torch.Tensor, TensorShape("n", "p")]
+2
View File
@@ -1315,6 +1315,7 @@ class MimoAudioEncoder(nn.Module):
Returns:
[T//group_size, group_size, audio_channels]
"""
T = audio.shape[0]
audio = audio[:, : self.audio_channels]
@@ -1355,6 +1356,7 @@ class MimoAudioEncoder(nn.Module):
Tuple of:
- audio_embeds: [total_tokens, out_hidden_size] concatenated embeddings
- item_token_lens: list of int, number of tokens per input item
"""
if self.audio_tokenizer is None:
raise RuntimeError(
+2 -2
View File
@@ -776,8 +776,7 @@ class MiMoV2Model(nn.Module, EagleModelMixin):
tp_rank: int,
tp_size: int,
) -> bool:
"""
The fused fp8 QKV projection weights and scale are stored separately.
"""The fused fp8 QKV projection weights and scale are stored separately.
Special care must be taken while sharding these tensors across TP ranks.
See _shard_fp8_qkv_proj for more details.
@@ -785,6 +784,7 @@ class MiMoV2Model(nn.Module, EagleModelMixin):
True if ``tensor`` was an fp8 qkv_proj weight/scale and was consumed
(caller should skip it); False otherwise, so the caller falls
through to its normal loading path.
"""
is_weight = (
name.endswith("qkv_proj.weight") and tensor.dtype == torch.float8_e4m3fn
+1 -1
View File
@@ -141,7 +141,7 @@ class MiMoV2MTPLayer(nn.Module):
class _MiMoV2MTPLayers(nn.Module):
"""Thin wrapper so parameter paths match checkpoint: model.mtp.layers.*"""
"""Thin wrapper so parameter paths match checkpoint: model.mtp.layers.*."""
def __init__(
self,
+14 -11
View File
@@ -275,14 +275,14 @@ class MiMoVisionAttention(nn.Module):
max_seqlen: torch.Tensor,
full_attn: bool = True,
) -> torch.Tensor:
"""
Args:
x: [seq_len, batch=1, embed_dim] (seq-first convention)
cu_seqlens: cumulative sequence lengths [num_seqs+1], int32
rotary_pos_emb_cos: [seq_len, qk_channels // 2]
rotary_pos_emb_sin: [seq_len, qk_channels // 2]
max_seqlen: maximum sequence length
full_attn: if True, full attention; if False, window attention
"""Args:
x: [seq_len, batch=1, embed_dim] (seq-first convention)
cu_seqlens: cumulative sequence lengths [num_seqs+1], int32
rotary_pos_emb_cos: [seq_len, qk_channels // 2]
rotary_pos_emb_sin: [seq_len, qk_channels // 2]
max_seqlen: maximum sequence length
full_attn: if True, full attention; if False, window attention
"""
# [seq_len, 1, embed_dim] -> QKV projection
qkv, _ = self.qkv(x) # [seq_len, 1, q_size + kv_size + kv_size]
@@ -531,6 +531,7 @@ class MiMoVisionTransformer(nn.Module):
Returns:
cos: [total_tokens, qk_channels // 2]
sin: [total_tokens, qk_channels // 2]
"""
cos_list, sin_list = [], []
for i in range(grid_thw.size(0)):
@@ -576,12 +577,12 @@ class MiMoVisionTransformer(nn.Module):
return torch.cat(cos_list, dim=0), torch.cat(sin_list, dim=0)
def forward(self, x: torch.Tensor, grid_thw: torch.Tensor) -> torch.Tensor:
"""
Args:
"""Args:
x: [total_tokens, C] pre-flattened patches
grid_thw: [num_images, 3] tensor of (t, h, w) for each image/video
Returns:
[merged_tokens, out_hidden_size]
"""
# Ensure grid_thw is a tensor
if not isinstance(grid_thw, torch.Tensor):
@@ -1556,8 +1557,10 @@ class MiMoV2OmniForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsQ
batch. **NOTE**: If mrope is enabled (default setting for
Qwen2.5-VL opensource models), the shape will be `(3, seq_len)`,
otherwise it will be `(seq_len,).
"""
intermediate_tensors: Intermediate tensors from prior forward pass.
inputs_embeds: Optional tensor of input embeddings.
"""
if intermediate_tensors is not None:
inputs_embeds = None
+8 -11
View File
@@ -131,13 +131,12 @@ if os.getenv("USE_FLAGOS") == "1":
class MiniCPMOAudioFeatureInputs(TensorSchema):
"""
Dimensions:
- bns: Batch size * number of audios * number of slices
- bn: Batch size * number of audios
- c: Number of channels
- l: Length
- s: Number of slices
"""Dimensions:
- bns: Batch size * number of audios * number of slices
- bn: Batch size * number of audios
- c: Number of channels
- l: Length
- s: Number of slices
"""
type: Literal["audio_features"] = "audio_features"
@@ -163,8 +162,7 @@ class MiniCPMOAudioFeatureInputs(TensorSchema):
class MiniCPMOAudioEmbeddingInputs(TensorSchema):
"""
Dimensions:
"""Dimensions:
- bn: Batch size * number of audios
- s: Number of slices
- h: Hidden size (must match language model backbone)
@@ -1008,8 +1006,7 @@ _MINICPMO_SUPPORT_VERSION = {
dummy_inputs=MiniCPMODummyInputsBuilder,
)
class MiniCPMO(MiniCPMOBaseModel, MiniCPMV2_6):
"""
MiniCPM-O model with audio support.
"""MiniCPM-O model with audio support.
Different versions use different LLM backbones:
- Version 2.6: Uses Qwen2
- Version 4.5: Uses Qwen3
+14 -22
View File
@@ -124,13 +124,12 @@ _MAX_FRAMES_PER_VIDEO = 16
class MiniCPMVImagePixelInputs(TensorSchema):
"""
Dimensions:
- bns: Batch size * number of images * number of slices
- bn: Batch size * number of images
- c: Number of channels
- h: Height
- w: Width
"""Dimensions:
- bns: Batch size * number of images * number of slices
- bn: Batch size * number of images
- c: Number of channels
- h: Height
- w: Width
"""
type: Literal["pixel_values"] = "pixel_values"
@@ -152,11 +151,10 @@ class MiniCPMVImagePixelInputs(TensorSchema):
class MiniCPMVImageEmbeddingInputs(TensorSchema):
"""
Dimensions:
- bn: Batch size * number of images
- ns: Number of slices
- hs: Hidden size (must match language model backbone)
"""Dimensions:
- bn: Batch size * number of images
- ns: Number of slices
- hs: Hidden size (must match language model backbone)
"""
type: Literal["image_embeds"]
@@ -307,8 +305,7 @@ class Resampler4_5(Resampler2_5):
def get_1d_sincos_pos_embed_from_temporal_size(
self, embed_dim: int, pos: np.ndarray
):
"""
embed_dim: output dimension for each position
"""embed_dim: output dimension for each position
pos: a list of positions to be encoded: size (M,)
out: (M, D)
"""
@@ -1219,8 +1216,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
"""
The abstract class of MiniCPMV can only be inherited, but cannot be
"""The abstract class of MiniCPMV can only be inherited, but cannot be
instantiated.
"""
@@ -1419,9 +1415,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
return loaded
def get_mm_mapping(self) -> MultiModelKeys:
"""
Get the module prefix in multimodal models
"""
"""Get the module prefix in multimodal models."""
return MultiModelKeys.from_string_field(
language_model="llm", connector="resampler", tower_model="vpm"
)
@@ -1494,7 +1488,6 @@ def _mcpmv_normalize_tgt_sizes(
slice_counts: list[int],
) -> torch.Tensor:
"""Normalize tgt_sizes to shape ``(total_slices, 2)``."""
if isinstance(tgt_sizes, list):
if not tgt_sizes:
tgt_sizes = torch.zeros((0, 2), dtype=torch.long)
@@ -2457,8 +2450,7 @@ _SUPPORT_VERSION = {
dummy_inputs=MiniCPMVDummyInputsBuilder,
)
class MiniCPMV(MiniCPMVBaseModel, SupportsMultiModal, SupportsLoRA):
"""
Different versions of MiniCPMV use different visual encoders and LLMs,
"""Different versions of MiniCPMV use different visual encoders and LLMs,
which is not conducive to the current integration logic of LoRA and
bitsandbytes in vLLM. Therefore, it is necessary to separate them.
"""

Some files were not shown because too many files have changed in this diff Show More