[Mypy] Fix mypy typing for L models (#54177)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-09-04 07:50:43 -04:00
committed by GitHub
parent 3ff4f02dfe
commit c615b1fd67
19 changed files with 255 additions and 89 deletions
-1
View File
@@ -113,7 +113,6 @@ SEPARATE_GROUPS = [
EXCLUDE = [ EXCLUDE = [
r"vllm/model_executor/models/[kK]", r"vllm/model_executor/models/[kK]",
r"vllm/model_executor/models/[lL]",
r"vllm/model_executor/models/[nN]", r"vllm/model_executor/models/[nN]",
r"vllm/model_executor/models/[oO]", r"vllm/model_executor/models/[oO]",
r"vllm/model_executor/models/[pP]", r"vllm/model_executor/models/[pP]",
+1
View File
@@ -317,6 +317,7 @@ class LagunaAttention(nn.Module):
# config.gating may be: # config.gating may be:
# - True / "per-element": one gate per (head, head_dim) channel # - True / "per-element": one gate per (head, head_dim) channel
# - "per-head": one gate per head, broadcast across head_dim # - "per-head": one gate per head, broadcast across head_dim
self.g_proj: ColumnParallelLinear | None
if self.gating: if self.gating:
# v5 LagunaConfig uses ``gating=True`` for per-head; older configs # v5 LagunaConfig uses ``gating=True`` for per-head; older configs
# used ``"per-head"``. Accept both. ``"per-element"`` (or legacy # used ``"per-head"``. Accept both. ``"per-element"`` (or legacy
+11 -15
View File
@@ -69,7 +69,9 @@ class DFlashLagunaModel(DFlashQwen3Model, EagleModelMixin):
prefix: str = "", prefix: str = "",
) -> None: ) -> None:
nn.Module.__init__(self) nn.Module.__init__(self)
self.config = vllm_config.speculative_config.draft_model_config.hf_config speculative_config = vllm_config.speculative_config
assert speculative_config is not None
self.config = speculative_config.draft_model_config.hf_config
self.vocab_size = self.config.vocab_size self.vocab_size = self.config.vocab_size
self.quant_config = get_draft_quant_config(vllm_config) self.quant_config = get_draft_quant_config(vllm_config)
@@ -216,24 +218,16 @@ class DFlashLagunaModel(DFlashQwen3Model, EagleModelMixin):
return all_k_normed return all_k_normed
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
if self.quant_config is not None:
weights = self.quant_config.get_cache_scale_mapper().apply(weights)
params_dict = dict(self.named_parameters()) params_dict = dict(self.named_parameters())
loaded_params: set[str] = set() loaded_params: set[str] = set()
for name, loaded_weight in weights: for name, loaded_weight in weights:
if self.quant_config is not None and (
scale_name := self.quant_config.get_cache_scale(name)
):
param = params_dict[scale_name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
loaded_weight = (
loaded_weight if loaded_weight.dim() == 0 else loaded_weight[0]
)
weight_loader(param, loaded_weight)
loaded_params.add(scale_name)
continue
if "scale" in name: if "scale" in name:
name = maybe_remap_kv_scale_name(name, params_dict) remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if name is None: if remapped_name is None:
continue continue
name = remapped_name
param = params_dict[name] param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader) weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight) weight_loader(param, loaded_weight)
@@ -244,7 +238,9 @@ class DFlashLagunaModel(DFlashQwen3Model, EagleModelMixin):
class DFlashLagunaForCausalLM(nn.Module, SupportsEagle3): class DFlashLagunaForCausalLM(nn.Module, SupportsEagle3):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
nn.Module.__init__(self) nn.Module.__init__(self)
self.config = vllm_config.speculative_config.draft_model_config.hf_config speculative_config = vllm_config.speculative_config
assert speculative_config is not None
self.config = speculative_config.draft_model_config.hf_config
if getattr(self.config, "draft_vocab_size", None) is None: if getattr(self.config, "draft_vocab_size", None) is None:
raise ValueError("Laguna DFlash config requires `draft_vocab_size`.") raise ValueError("Laguna DFlash config requires `draft_vocab_size`.")
self.has_own_embed_tokens = False self.has_own_embed_tokens = False
+21
View File
@@ -3,6 +3,7 @@
import itertools import itertools
import math import math
import typing
from collections.abc import Iterable, Mapping, Sequence from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Any, Literal from typing import Annotated, Any, Literal
@@ -17,6 +18,7 @@ from transformers.models.lfm2_vl.image_processing_lfm2_vl_fast import (
find_closest_aspect_ratio, find_closest_aspect_ratio,
round_by_factor, round_by_factor,
) )
from typing_extensions import Buffer
from vllm.config import VllmConfig from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions from vllm.config.multimodal import BaseDummyOptions
@@ -229,6 +231,13 @@ class Lfm2VLProcessingInfo(BaseProcessingInfo):
"max_image_tokens", image_processor.max_image_tokens "max_image_tokens", image_processor.max_image_tokens
) )
tile_size = mm_kwargs.get("tile_size", image_processor.tile_size) tile_size = mm_kwargs.get("tile_size", image_processor.tile_size)
assert isinstance(downsample_factor, int)
assert isinstance(encoder_patch_size, int)
assert isinstance(max_pixels_tolerance, int | float)
assert isinstance(min_tiles, int)
assert isinstance(max_tiles, int)
assert isinstance(max_image_tokens, int)
assert isinstance(tile_size, int)
do_image_splitting = not min_tiles == max_tiles == 1 do_image_splitting = not min_tiles == max_tiles == 1
is_image_large = self._is_image_too_large( is_image_large = self._is_image_too_large(
@@ -337,6 +346,9 @@ class Lfm2VLProcessingInfo(BaseProcessingInfo):
"encoder_patch_size", image_processor.encoder_patch_size "encoder_patch_size", image_processor.encoder_patch_size
) )
tile_size = mm_kwargs.get("tile_size", image_processor.tile_size) tile_size = mm_kwargs.get("tile_size", image_processor.tile_size)
assert isinstance(downsample_factor, int)
assert isinstance(encoder_patch_size, int)
assert isinstance(tile_size, int)
thumbnail_height_patches = int(spatial_shapes[-1][0].item()) thumbnail_height_patches = int(spatial_shapes[-1][0].item())
thumbnail_width_patches = int(spatial_shapes[-1][1].item()) thumbnail_width_patches = int(spatial_shapes[-1][1].item())
@@ -672,6 +684,7 @@ class Lfm2VLForConditionalGeneration(
super().__init__() super().__init__()
config: Lfm2VlConfig = vllm_config.model_config.hf_config config: Lfm2VlConfig = vllm_config.model_config.hf_config
multimodal_config = vllm_config.model_config.multimodal_config multimodal_config = vllm_config.model_config.multimodal_config
assert multimodal_config is not None
vision_config = config.vision_config vision_config = config.vision_config
quant_config = vllm_config.quant_config quant_config = vllm_config.quant_config
@@ -915,6 +928,14 @@ class Lfm2VLForConditionalGeneration(
"min_image_tokens", "min_image_tokens",
getattr(self.config, "min_image_tokens", None) or 64, getattr(self.config, "min_image_tokens", None) or 64,
) )
if not isinstance(
value,
(str, Buffer, typing.SupportsInt, typing.SupportsIndex),
):
raise TypeError(
"int() argument must be a string, a bytes-like object "
f"or a real number, not '{type(value).__name__}'"
)
return max(1, int(value)) return max(1, int(value))
def _get_lfm2vl_item_tile_slices( def _get_lfm2vl_item_tile_slices(
+1 -1
View File
@@ -49,7 +49,7 @@ class LightOnOCRProcessingInfo(Mistral3ProcessingInfo):
return PixtralHFEncoderInfo(self.get_hf_config()) return PixtralHFEncoderInfo(self.get_hf_config())
class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[LightOnOCRProcessingInfo]): class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts) return self.dummy_inputs.get_dummy_text(mm_counts)
+15 -2
View File
@@ -26,6 +26,7 @@
from collections.abc import Iterable from collections.abc import Iterable
from itertools import islice from itertools import islice
from typing import TYPE_CHECKING
import torch import torch
from torch import nn from torch import nn
@@ -544,13 +545,25 @@ class LlamaForCausalLM(
return loader.load_weights(weights) return loader.load_weights(weights)
class LlamaBidirectionalForSequenceClassification(as_seq_cls_model(LlamaForCausalLM)): if TYPE_CHECKING:
_LlamaBidirectionalForSequenceClassificationBase = LlamaForCausalLM
_LlamaBidirectionalModelBase = LlamaForCausalLM
else:
_LlamaBidirectionalForSequenceClassificationBase = as_seq_cls_model(
LlamaForCausalLM
)
_LlamaBidirectionalModelBase = as_embedding_model(LlamaForCausalLM)
class LlamaBidirectionalForSequenceClassification(
_LlamaBidirectionalForSequenceClassificationBase
):
# This class sets the correct attention type and pooling type # This class sets the correct attention type and pooling type
# through LlamaBidirectionalConfig. # through LlamaBidirectionalConfig.
pass pass
class LlamaBidirectionalModel(as_embedding_model(LlamaForCausalLM)): class LlamaBidirectionalModel(_LlamaBidirectionalModelBase):
# This class sets the correct attention type and pooling type # This class sets the correct attention type and pooling type
# through LlamaBidirectionalConfig. # through LlamaBidirectionalConfig.
pass pass
+5 -3
View File
@@ -18,7 +18,7 @@
# limitations under the License. # limitations under the License.
"""Inference-only LLaMA model compatible with HuggingFace weights.""" """Inference-only LLaMA model compatible with HuggingFace weights."""
from collections.abc import Iterable from collections.abc import Callable, Iterable
import torch import torch
from torch import nn from torch import nn
@@ -576,6 +576,7 @@ class Llama4Model(LlamaModel):
params_dict = dict(self.named_parameters()) params_dict = dict(self.named_parameters())
# The module parameters that have been loaded. # The module parameters that have been loaded.
loaded_params: set[str] = set() loaded_params: set[str] = set()
weight_loader: Callable[..., object]
# Iterate over all the weights and load them into module parameters. # Iterate over all the weights and load them into module parameters.
for name, loaded_weight in weights: for name, loaded_weight in weights:
@@ -612,9 +613,10 @@ class Llama4Model(LlamaModel):
# quant config's `get_cache_scale_mapper` does not cover # quant config's `get_cache_scale_mapper` does not cover
# (idempotent for names already renamed by the mapper). # (idempotent for names already renamed by the mapper).
if name.endswith("scale"): if name.endswith("scale"):
name = maybe_remap_kv_scale_name(name, params_dict) remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if name is None: if remapped_name is None:
continue continue
name = remapped_name
# Load the weight into the module parameter with corresponding # Load the weight into the module parameter with corresponding
# shard id and exit the for loop and the else block. # shard id and exit the for loop and the else block.
+22 -4
View File
@@ -17,6 +17,7 @@
# limitations under the License. # limitations under the License.
from collections.abc import Iterable from collections.abc import Iterable
from typing import TYPE_CHECKING
import torch import torch
import torch.nn as nn import torch.nn as nn
@@ -46,6 +47,19 @@ from .utils import (
logger = init_logger(__name__) logger = init_logger(__name__)
if TYPE_CHECKING:
class _EagleLlama4ForCausalLMBase(nn.Module):
def set_moe_parameters(self) -> None: ...
def permute_qk_weight_for_rotary(
self, name: str, loaded_weight: torch.Tensor
) -> tuple[str, torch.Tensor]: ...
else:
_EagleLlama4ForCausalLMBase = Llama4ForCausalLM
@support_torch_compile @support_torch_compile
class LlamaModel(nn.Module): class LlamaModel(nn.Module):
hf_to_vllm_mapper = WeightsMapper( hf_to_vllm_mapper = WeightsMapper(
@@ -68,7 +82,9 @@ class LlamaModel(nn.Module):
quant_config: QuantizationConfig | None = None, quant_config: QuantizationConfig | None = None,
) -> None: ) -> None:
super().__init__() super().__init__()
self.config = vllm_config.speculative_config.draft_model_config.hf_config speculative_config = vllm_config.speculative_config
assert speculative_config is not None
self.config = speculative_config.draft_model_config.hf_config
self.validate_and_update_config(start_layer_id, quant_config) self.validate_and_update_config(start_layer_id, quant_config)
self.vocab_size = self.config.vocab_size self.vocab_size = self.config.vocab_size
self.embed_tokens = VocabParallelEmbedding( self.embed_tokens = VocabParallelEmbedding(
@@ -152,16 +168,18 @@ class LlamaModel(nn.Module):
} }
class EagleLlama4ForCausalLM(Llama4ForCausalLM, SupportsMultiModalEmbeddings): class EagleLlama4ForCausalLM(_EagleLlama4ForCausalLMBase, SupportsMultiModalEmbeddings):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
nn.Module.__init__(self) nn.Module.__init__(self)
self.config = vllm_config.speculative_config.draft_model_config.hf_config speculative_config = vllm_config.speculative_config
assert speculative_config is not None
self.config = speculative_config.draft_model_config.hf_config
target_layer_num = vllm_config.model_config.get_num_layers( target_layer_num = vllm_config.model_config.get_num_layers(
vllm_config.parallel_config vllm_config.parallel_config
) )
# draft model quantization config may differ from target model # draft model quantization config may differ from target model
quant_config = VllmConfig.get_quantization_config( quant_config = VllmConfig.get_quantization_config(
vllm_config.speculative_config.draft_model_config, vllm_config.load_config speculative_config.draft_model_config, vllm_config.load_config
) )
self.model = LlamaModel( self.model = LlamaModel(
vllm_config=vllm_config, vllm_config=vllm_config,
+22 -5
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable from collections.abc import Iterable
from typing import TYPE_CHECKING
import torch import torch
import torch.nn as nn import torch.nn as nn
@@ -14,7 +15,10 @@ from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.models.llama import LlamaDecoderLayer, LlamaForCausalLM from vllm.model_executor.models.llama import (
LlamaDecoderLayer as BaseLlamaDecoderLayer,
)
from vllm.model_executor.models.llama import LlamaForCausalLM
from .utils import ( from .utils import (
AutoWeightsLoader, AutoWeightsLoader,
@@ -27,7 +31,16 @@ from .utils import (
logger = init_logger(__name__) logger = init_logger(__name__)
class LlamaDecoderLayer(LlamaDecoderLayer): if TYPE_CHECKING:
class _EagleLlamaForCausalLMBase(nn.Module):
pass
else:
_EagleLlamaForCausalLMBase = LlamaForCausalLM
class LlamaDecoderLayer(BaseLlamaDecoderLayer):
def __init__( def __init__(
self, self,
vllm_config: VllmConfig, vllm_config: VllmConfig,
@@ -69,7 +82,9 @@ class LlamaModel(nn.Module):
start_layer_id: int = 0, start_layer_id: int = 0,
) -> None: ) -> None:
super().__init__() super().__init__()
self.config = vllm_config.speculative_config.draft_model_config.hf_config speculative_config = vllm_config.speculative_config
assert speculative_config is not None
self.config = speculative_config.draft_model_config.hf_config
self.vocab_size = self.config.vocab_size self.vocab_size = self.config.vocab_size
# Get drafter's quantization config # Get drafter's quantization config
@@ -128,10 +143,12 @@ class LlamaModel(nn.Module):
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class EagleLlamaForCausalLM(LlamaForCausalLM): class EagleLlamaForCausalLM(_EagleLlamaForCausalLMBase):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
nn.Module.__init__(self) nn.Module.__init__(self)
self.config = vllm_config.speculative_config.draft_model_config.hf_config speculative_config = vllm_config.speculative_config
assert speculative_config is not None
self.config = speculative_config.draft_model_config.hf_config
# Ensure draft_vocab_size is set # Ensure draft_vocab_size is set
# default to the base vocab size when absent # default to the base vocab size when absent
if getattr(self.config, "draft_vocab_size", None) is None: if getattr(self.config, "draft_vocab_size", None) is None:
+44 -7
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable from collections.abc import Iterable
from typing import TYPE_CHECKING
import torch import torch
import torch.nn as nn import torch.nn as nn
@@ -18,7 +19,14 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead, ParallelLMHead,
VocabParallelEmbedding, VocabParallelEmbedding,
) )
from vllm.model_executor.models.llama import LlamaDecoderLayer, LlamaForCausalLM from vllm.model_executor.models.llama import (
LlamaAttention,
LlamaForCausalLM,
LlamaMLP,
)
from vllm.model_executor.models.llama import (
LlamaDecoderLayer as BaseLlamaDecoderLayer,
)
from vllm.multimodal.inputs import NestedTensors from vllm.multimodal.inputs import NestedTensors
from .utils import ( from .utils import (
@@ -32,7 +40,31 @@ from .utils import (
logger = init_logger(__name__) logger = init_logger(__name__)
class LlamaDecoderLayer(LlamaDecoderLayer): if TYPE_CHECKING:
class _Eagle3LlamaDecoderLayerBase(nn.Module):
hidden_size: int
self_attn: LlamaAttention
mlp: LlamaMLP
input_layernorm: RMSNorm
post_attention_layernorm: RMSNorm
def __init__(
self,
vllm_config: VllmConfig,
prefix: str = "",
config: LlamaConfig | None = None,
) -> None: ...
class _Eagle3LlamaForCausalLMBase(nn.Module):
pass
else:
_Eagle3LlamaDecoderLayerBase = BaseLlamaDecoderLayer
_Eagle3LlamaForCausalLMBase = LlamaForCausalLM
class LlamaDecoderLayer(_Eagle3LlamaDecoderLayerBase):
def __init__( def __init__(
self, self,
vllm_config: VllmConfig, vllm_config: VllmConfig,
@@ -136,7 +168,9 @@ class LlamaModel(nn.Module):
prefix: str = "", prefix: str = "",
) -> None: ) -> None:
super().__init__() super().__init__()
self.config = vllm_config.speculative_config.draft_model_config.hf_config speculative_config = vllm_config.speculative_config
assert speculative_config is not None
self.config = speculative_config.draft_model_config.hf_config
self.vocab_size = self.config.vocab_size self.vocab_size = self.config.vocab_size
# Get drafter's quantization config # Get drafter's quantization config
@@ -187,7 +221,7 @@ class LlamaModel(nn.Module):
self.fc_input_size = target_hidden_size * self.num_aux_hidden_states self.fc_input_size = target_hidden_size * self.num_aux_hidden_states
if self.norm_before_fc: if self.norm_before_fc:
self.input_norm = RMSNorm( self.input_norm: RMSNorm | None = RMSNorm(
self.fc_input_size, self.fc_input_size,
eps=self.config.rms_norm_eps, eps=self.config.rms_norm_eps,
) )
@@ -269,10 +303,12 @@ class LlamaModel(nn.Module):
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class Eagle3LlamaForCausalLM(LlamaForCausalLM): class Eagle3LlamaForCausalLM(_Eagle3LlamaForCausalLMBase):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
nn.Module.__init__(self) nn.Module.__init__(self)
self.config = vllm_config.speculative_config.draft_model_config.hf_config speculative_config = vllm_config.speculative_config
assert speculative_config is not None
self.config = speculative_config.draft_model_config.hf_config
# Ensure draft_vocab_size is set # Ensure draft_vocab_size is set
# default to the base vocab size when absent # default to the base vocab size when absent
if getattr(self.config, "draft_vocab_size", None) is None: if getattr(self.config, "draft_vocab_size", None) is None:
@@ -306,7 +342,7 @@ class Eagle3LlamaForCausalLM(LlamaForCausalLM):
requires_grad=False, requires_grad=False,
) )
self.use_parallel_drafting = vllm_config.speculative_config.parallel_drafting self.use_parallel_drafting = speculative_config.parallel_drafting
if self.use_parallel_drafting: if self.use_parallel_drafting:
self.register_buffer( self.register_buffer(
@@ -365,6 +401,7 @@ class Eagle3LlamaForCausalLM(LlamaForCausalLM):
# combine multiple auxiliary hidden states returned by eagle3 # combine multiple auxiliary hidden states returned by eagle3
if self.model.norm_before_fc: if self.model.norm_before_fc:
assert self.model.input_norm is not None
hidden_states = self.model.input_norm(hidden_states) hidden_states = self.model.input_norm(hidden_states)
# `norm_before_fc` adds a single RMSNorm before the FC layer, whereas `fc_norm` # `norm_before_fc` adds a single RMSNorm before the FC layer, whereas `fc_norm`
+9 -7
View File
@@ -3,7 +3,7 @@
from abc import abstractmethod from abc import abstractmethod
from collections.abc import Iterable, Mapping, Sequence from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Final, Literal, Protocol, TypeAlias, TypeVar from typing import Annotated, Literal, Protocol, TypeAlias, TypeVar
import torch import torch
import torch.nn as nn import torch.nn as nn
@@ -159,14 +159,14 @@ class LlavaMultiModalProjector(nn.Module):
class LlavaLikeConfig(Protocol): class LlavaLikeConfig(Protocol):
vision_config: Final[PretrainedConfig] vision_config: PretrainedConfig
image_token_index: Final[int] image_token_index: int
vision_feature_select_strategy: Final[str] vision_feature_select_strategy: str
vision_feature_layer: Final[int | list[int]] vision_feature_layer: int | list[int]
class LlavaLikeProcessor(Protocol): class LlavaLikeProcessor(Protocol):
image_token: Final[str] image_token: str
class BaseLlavaProcessingInfo(BaseProcessingInfo): class BaseLlavaProcessingInfo(BaseProcessingInfo):
@@ -286,6 +286,7 @@ class BaseLlavaMultiModalProcessor(BaseMultiModalProcessor[_I]):
if isinstance(images, ImageEmbeddingItems): if isinstance(images, ImageEmbeddingItems):
num_image_tokens = images.get_feature_size(item_idx) num_image_tokens = images.get_feature_size(item_idx)
else: else:
assert isinstance(images, ImageProcessorItems)
image_size = images.get_image_size(item_idx) image_size = images.get_image_size(item_idx)
num_image_tokens = self.info.get_num_image_tokens( num_image_tokens = self.info.get_num_image_tokens(
image_width=image_size.width, image_width=image_size.width,
@@ -372,6 +373,7 @@ class PixtralHFMultiModalProcessor(BaseMultiModalProcessor[PixtralHFProcessingIn
image_end_id = vocab[processor.image_end_token] image_end_id = vocab[processor.image_end_token]
assert isinstance(hf_config.vision_config, PixtralVisionConfig) assert isinstance(hf_config.vision_config, PixtralVisionConfig)
assert isinstance(hf_config, LlavaConfig)
encoder_info = PixtralHFEncoderInfo(hf_config) encoder_info = PixtralHFEncoderInfo(hf_config)
def get_replacement(item_idx: int): def get_replacement(item_idx: int):
@@ -642,7 +644,7 @@ class LlavaForConditionalGeneration(
self, self,
image_input: LlavaImageInputs, image_input: LlavaImageInputs,
) -> torch.Tensor | tuple[torch.Tensor, ...]: ) -> torch.Tensor | tuple[torch.Tensor, ...]:
if image_input["type"] == "image_embeds": if isinstance(image_input, LlavaImageEmbeddingInputs):
return image_input["data"] return image_input["data"]
image_features = self._process_image_pixels(image_input) image_features = self._process_image_pixels(image_input)
+3 -3
View File
@@ -3,7 +3,7 @@
from abc import abstractmethod from abc import abstractmethod
from collections.abc import Iterable, Mapping from collections.abc import Iterable, Mapping
from typing import Annotated, Final, Literal, Protocol, TypeAlias, TypeVar from typing import Annotated, Literal, Protocol, TypeAlias, TypeVar
import torch import torch
import torch.nn as nn import torch.nn as nn
@@ -88,7 +88,7 @@ LlavaNextImageInputs: TypeAlias = (
class LlavaNextLikeConfig(LlavaLikeConfig, Protocol): class LlavaNextLikeConfig(LlavaLikeConfig, Protocol):
image_grid_pinpoints: Final[list[list[int]]] image_grid_pinpoints: list[list[int]]
class LlavaNextProcessingInfo(BaseLlavaProcessingInfo): class LlavaNextProcessingInfo(BaseLlavaProcessingInfo):
@@ -483,7 +483,7 @@ class LlavaNextForConditionalGeneration(
self, self,
image_input: LlavaNextImageInputs, image_input: LlavaNextImageInputs,
) -> torch.Tensor | list[torch.Tensor]: ) -> torch.Tensor | list[torch.Tensor]:
if image_input["type"] == "image_embeds": if isinstance(image_input, LlavaNextImageEmbeddingInputs):
return image_input["data"] return image_input["data"]
patch_embeddings = self._process_image_pixels(image_input) patch_embeddings = self._process_image_pixels(image_input)
@@ -10,7 +10,7 @@ import torch.nn as nn
from transformers import BatchFeature, LlavaNextVideoConfig, LlavaNextVideoProcessor from transformers import BatchFeature, LlavaNextVideoConfig, LlavaNextVideoProcessor
from vllm.config import VllmConfig from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
from vllm.inputs import MultiModalDataDict from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.models.clip import CLIPVisionModel from vllm.model_executor.models.clip import CLIPVisionModel
@@ -181,6 +181,7 @@ class LlavaNextVideoDummyInputsBuilder(
) )
video_overrides = mm_options.get("video") video_overrides = mm_options.get("video")
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
return { return {
"video": self._get_dummy_videos( "video": self._get_dummy_videos(
@@ -220,6 +221,7 @@ class LlavaNextVideoMultiModalProcessor(
if isinstance(videos, VideoEmbeddingItems): if isinstance(videos, VideoEmbeddingItems):
num_video_tokens = videos.get_feature_size(item_idx) num_video_tokens = videos.get_feature_size(item_idx)
else: else:
assert isinstance(videos, VideoProcessorItems)
image_size = videos.get_frame_size(item_idx) image_size = videos.get_frame_size(item_idx)
num_video_tokens = self.info.get_num_video_tokens( num_video_tokens = self.info.get_num_video_tokens(
image_width=image_size.width, image_width=image_size.width,
+21 -9
View File
@@ -3,7 +3,7 @@
import math import math
from collections.abc import Iterable, Mapping, Sequence from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Final, Literal, Protocol, TypeAlias from typing import Annotated, Literal, Protocol, TypeAlias, TypedDict
import torch import torch
import torch.nn as nn import torch.nn as nn
@@ -14,7 +14,7 @@ from transformers.models.llava_onevision.modeling_llava_onevision import (
) )
from vllm.config import VllmConfig from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
from vllm.inputs import MultiModalDataDict from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.layers.activation import get_act_fn
from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -123,7 +123,12 @@ LlavaOnevisionMultiInputs: TypeAlias = (
class LlavaOnevisionLikeConfig(LlavaNextLikeConfig, Protocol): class LlavaOnevisionLikeConfig(LlavaNextLikeConfig, Protocol):
video_token_index: Final[int] video_token_index: int
class LlavaOnevisionInputsByModality(TypedDict, total=False):
image: LlavaOnevisionImageInputs | None
video: LlavaOnevisionVideoPixelInputs | None
class LlavaOnevisionProcessingInfo(LlavaNextProcessingInfo): class LlavaOnevisionProcessingInfo(LlavaNextProcessingInfo):
@@ -288,6 +293,7 @@ class LlavaOnevisionDummyInputsBuilder(
image_overrides = mm_options.get("image") image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video") video_overrides = mm_options.get("video")
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
return { return {
"image": self._get_dummy_images( "image": self._get_dummy_images(
@@ -417,6 +423,7 @@ class LlavaOnevisionMultiModalProcessor(
if isinstance(videos, VideoEmbeddingItems): if isinstance(videos, VideoEmbeddingItems):
num_video_tokens = videos.get_feature_size(item_idx) num_video_tokens = videos.get_feature_size(item_idx)
else: else:
assert isinstance(videos, VideoProcessorItems)
image_size = videos.get_frame_size(item_idx) image_size = videos.get_frame_size(item_idx)
num_video_tokens = self.info.get_num_video_tokens( num_video_tokens = self.info.get_num_video_tokens(
image_width=image_size.width, image_width=image_size.width,
@@ -570,8 +577,10 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
}, },
) )
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: def _parse_and_validate_multimodal_inputs(
mm_input_by_modality = {} self, **kwargs: object
) -> LlavaOnevisionInputsByModality:
mm_input_by_modality = LlavaOnevisionInputsByModality()
# Preserve the order of modalities if there are multiple of them # Preserve the order of modalities if there are multiple of them
# from the order of kwargs. # from the order of kwargs.
@@ -749,7 +758,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
self, self,
image_input: LlavaOnevisionImageInputs, image_input: LlavaOnevisionImageInputs,
) -> torch.Tensor | list[torch.Tensor]: ) -> torch.Tensor | list[torch.Tensor]:
if image_input["type"] == "image_embeds": if isinstance(image_input, LlavaOnevisionImageEmbeddingInputs):
return image_input["data"] return image_input["data"]
patch_embeddings = self._process_image_pixels(image_input) patch_embeddings = self._process_image_pixels(image_input)
@@ -860,12 +869,15 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
# NOTE: It is important to iterate over the keys in this dictionary # NOTE: It is important to iterate over the keys in this dictionary
# to preserve the order of the modalities. # to preserve the order of the modalities.
for modality in mm_input_by_modality: for modality in mm_input_by_modality:
multimodal_input = mm_input_by_modality[modality]
if modality == "image": if modality == "image":
image_embeddings = self._process_image_input(multimodal_input) image_input = mm_input_by_modality["image"]
assert image_input is not None
image_embeddings = self._process_image_input(image_input)
multimodal_embeddings += tuple(image_embeddings) multimodal_embeddings += tuple(image_embeddings)
if modality == "video": if modality == "video":
video_embeddings = self._process_video_pixels(multimodal_input) video_input = mm_input_by_modality["video"]
assert video_input is not None
video_embeddings = self._process_video_pixels(video_input)
multimodal_embeddings += tuple(video_embeddings) multimodal_embeddings += tuple(video_embeddings)
return multimodal_embeddings return multimodal_embeddings
+55 -18
View File
@@ -24,12 +24,15 @@ from __future__ import annotations
import hashlib import hashlib
import importlib import importlib
from collections.abc import Callable, Iterable, Mapping, Sequence from collections.abc import Callable, Iterable, Mapping, Sequence, Sized
from functools import lru_cache from functools import lru_cache
from typing import ( from typing import (
Annotated, Annotated,
Any, Any,
Literal, Literal,
SupportsIndex,
SupportsInt,
TypedDict,
) )
import numpy as np import numpy as np
@@ -45,6 +48,7 @@ from transformers.dynamic_module_utils import (
) )
from transformers.models.qwen2_vl import Qwen2VLImageProcessor from transformers.models.qwen2_vl import Qwen2VLImageProcessor
from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
from typing_extensions import Buffer
from vllm.compilation.decorators import ( from vllm.compilation.decorators import (
should_torch_compile_mm_encoder, should_torch_compile_mm_encoder,
@@ -730,6 +734,11 @@ LlavaOnevision2ImageInputs = (
) )
class LlavaOnevision2InputsByModality(TypedDict, total=False):
images: LlavaOnevision2ImageInputs | None
videos: LlavaOnevision2VideoPixelInputs | None
class LlavaOnevision2VisionRotaryEmbedding(nn.Module): class LlavaOnevision2VisionRotaryEmbedding(nn.Module):
"""3-D rotary frequency constructor with 4:6:6 (T:H:W) split. """3-D rotary frequency constructor with 4:6:6 (T:H:W) split.
@@ -1468,7 +1477,7 @@ class LlavaOnevision2DummyInputsBuilder(
n_img = mm_counts.get("image", 0) n_img = mm_counts.get("image", 0)
n_vid = mm_counts.get("video", 0) n_vid = mm_counts.get("video", 0)
w, h = self.info.get_image_size_with_most_features() w, h = self.info.get_image_size_with_most_features()
out: MultiModalDataDict = {} out: dict[str, object] = {}
if n_img: if n_img:
out["image"] = self._get_dummy_images(width=w, height=h, num_images=n_img) out["image"] = self._get_dummy_images(width=w, height=h, num_images=n_img)
if n_vid: if n_vid:
@@ -1586,6 +1595,8 @@ class LlavaOnevision2MultiModalProcessor(
# array, or tensor, and ``and <array>`` would raise on the ambiguous # array, or tensor, and ``and <array>`` would raise on the ambiguous
# truth value of a multi-element array. # truth value of a multi-element array.
_videos = mm_data.get("videos") _videos = mm_data.get("videos")
if _videos is not None and not isinstance(_videos, Sized):
raise TypeError(f"object of type '{type(_videos).__name__}' has no len()")
videos_present = _videos is not None and len(_videos) > 0 videos_present = _videos is not None and len(_videos) > 0
codec_video_paths = ( codec_video_paths = (
@@ -1666,15 +1677,24 @@ class LlavaOnevision2MultiModalProcessor(
# provided by the backend's ``compute_frames_index_to_sample``; SSRF / # provided by the backend's ``compute_frames_index_to_sample``; SSRF /
# local-file gating is enforced by the connector before decoding. # local-file gating is enforced by the connector before decoding.
if videos_present: if videos_present:
timestamp_decimals = int( timestamp_decimals = hf_processor_mm_kwargs.get(
hf_processor_mm_kwargs.get( "timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS
"timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS
)
) )
if not isinstance(
timestamp_decimals, (str, Buffer, SupportsInt, SupportsIndex)
):
raise TypeError(
"int() argument must be a string, a bytes-like object "
f"or a real number, not '{type(timestamp_decimals).__name__}'"
)
timestamp_decimals = int(timestamp_decimals)
per_video_frames: list[list[Image.Image]] = [] per_video_frames: list[list[Image.Image]] = []
per_video_timestamps: list[list[float]] = [] per_video_timestamps: list[list[float]] = []
for item in mm_data["videos"]: videos = mm_data["videos"]
if not isinstance(videos, Iterable):
raise TypeError(f"Unsupported videos input: {type(videos)}")
for item in videos:
pil_frames, timestamps = _frame_video_to_pil_and_timestamps(item) pil_frames, timestamps = _frame_video_to_pil_and_timestamps(item)
per_video_frames.append(pil_frames) per_video_frames.append(pil_frames)
per_video_timestamps.append(timestamps) per_video_timestamps.append(timestamps)
@@ -1902,18 +1922,21 @@ class LlavaOnevision2MultiModalProcessor(
def get_image_replacement(item_idx: int): def get_image_replacement(item_idx: int):
out_item = out_mm_kwargs["image"][item_idx] out_item = out_mm_kwargs["image"][item_idx]
grid_thw = out_item["image_grid_thw"].data grid_thw = out_item["image_grid_thw"].data
assert isinstance(grid_thw, torch.Tensor)
n = int(grid_thw.prod(-1).sum()) // merge_length n = int(grid_thw.prod(-1).sum()) // merge_length
return [image_pad_id] * n return [image_pad_id] * n
def get_video_replacement(item_idx: int): def get_video_replacement(item_idx: int):
out_item = out_mm_kwargs["video"][item_idx] out_item = out_mm_kwargs["video"][item_idx]
grid_thw = out_item["video_grid_thw"].data grid_thw = out_item["video_grid_thw"].data
assert isinstance(grid_thw, torch.Tensor)
is_codec_field = out_item.get("video_is_codec") is_codec_field = out_item.get("video_is_codec")
is_codec = ( if is_codec_field is not None:
bool(int(is_codec_field.data.item())) is_codec_data = is_codec_field.data
if is_codec_field is not None assert isinstance(is_codec_data, torch.Tensor)
else False is_codec = bool(int(is_codec_data.item()))
) else:
is_codec = False
tokens: list[int] = [] tokens: list[int] = []
if is_codec: if is_codec:
@@ -1924,6 +1947,8 @@ class LlavaOnevision2MultiModalProcessor(
# run, emit ``<sec seconds><|vision_start|><pad*N><|vision_end|>\n``. # run, emit ``<sec seconds><|vision_start|><pad*N><|vision_end|>\n``.
patch_positions = out_item["patch_positions_videos"].data patch_positions = out_item["patch_positions_videos"].data
fps_t = out_item["codec_fps"].data fps_t = out_item["codec_fps"].data
assert isinstance(patch_positions, torch.Tensor)
assert isinstance(fps_t, torch.Tensor)
fps = float(int(fps_t.item())) / 1_000_000_000.0 fps = float(int(fps_t.item())) / 1_000_000_000.0
runs = _codec_timestamp_runs( runs = _codec_timestamp_runs(
patch_positions, fps, image_processor.merge_size patch_positions, fps, image_processor.merge_size
@@ -1938,6 +1963,7 @@ class LlavaOnevision2MultiModalProcessor(
tokens.extend(newline_ids) tokens.extend(newline_ids)
else: else:
timestamps = out_item["frame_timestamps"].data timestamps = out_item["frame_timestamps"].data
assert isinstance(timestamps, torch.Tensor)
T_total = int(grid_thw.shape[0]) T_total = int(grid_thw.shape[0])
for t in range(T_total): for t in range(T_total):
sec = float(timestamps[t].item()) sec = float(timestamps[t].item())
@@ -2121,7 +2147,7 @@ class LlavaOnevision2ForConditionalGeneration(
grid_thw = image_input["image_grid_thw"] grid_thw = image_input["image_grid_thw"]
assert grid_thw.ndim == 2 assert grid_thw.ndim == 2
if image_input["type"] == "image_embeds": if isinstance(image_input, LlavaOnevision2ImageEmbeddingInputs):
image_embeds = image_input["image_embeds"] image_embeds = image_input["image_embeds"]
else: else:
image_embeds = self.visual( image_embeds = self.visual(
@@ -2166,8 +2192,13 @@ class LlavaOnevision2ForConditionalGeneration(
) )
if isinstance(video_num_frames, list): if isinstance(video_num_frames, list):
video_num_frames = torch.cat([v.flatten() for v in video_num_frames]) video_num_frames = torch.cat([v.flatten() for v in video_num_frames])
else: elif isinstance(video_num_frames, torch.Tensor):
video_num_frames = video_num_frames.flatten() video_num_frames = video_num_frames.flatten()
else:
raise TypeError(
"video_num_frames must be a tensor or a list of tensors, "
f"got {type(video_num_frames)}"
)
return LlavaOnevision2VideoPixelInputs( return LlavaOnevision2VideoPixelInputs(
type="pixel_values_videos", type="pixel_values_videos",
pixel_values_videos=pixel_values_videos, pixel_values_videos=pixel_values_videos,
@@ -2202,8 +2233,10 @@ class LlavaOnevision2ForConditionalGeneration(
cursor += n cursor += n
return video_embeds.split(sizes) return video_embeds.split(sizes)
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: def _parse_and_validate_multimodal_inputs(
modalities = {} self, **kwargs: object
) -> LlavaOnevision2InputsByModality:
modalities = LlavaOnevision2InputsByModality()
for key in kwargs: for key in kwargs:
if key in ("pixel_values", "image_embeds") and "images" not in modalities: if key in ("pixel_values", "image_embeds") and "images" not in modalities:
modalities["images"] = self._parse_and_validate_image_input(**kwargs) modalities["images"] = self._parse_and_validate_image_input(**kwargs)
@@ -2221,9 +2254,13 @@ class LlavaOnevision2ForConditionalGeneration(
multimodal_embeddings: tuple[torch.Tensor, ...] = () multimodal_embeddings: tuple[torch.Tensor, ...] = ()
for modality in modalities: for modality in modalities:
if modality == "images": if modality == "images":
multimodal_embeddings += self._process_image_input(modalities["images"]) image_input = modalities["images"]
assert image_input is not None
multimodal_embeddings += self._process_image_input(image_input)
elif modality == "videos": elif modality == "videos":
multimodal_embeddings += self._process_video_input(modalities["videos"]) video_input = modalities["videos"]
assert video_input is not None
multimodal_embeddings += self._process_video_input(video_input)
return multimodal_embeddings return multimodal_embeddings
def get_input_embeddings( def get_input_embeddings(
+12 -7
View File
@@ -83,6 +83,8 @@ logger = init_logger(__name__)
class FlashConfig(PretrainedConfig): class FlashConfig(PretrainedConfig):
"""Flash model configuration.""" """Flash model configuration."""
moe_intermediate_size: int
model_type = "longcat_flash" model_type = "longcat_flash"
keys_to_ignore_at_inference = ["past_key_values"] keys_to_ignore_at_inference = ["past_key_values"]
@@ -604,7 +606,7 @@ class FlashModel(nn.Module):
else: else:
is_expert_weight = False is_expert_weight = False
for mapping in expert_params_mapping: for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping param_name, weight_name, expert_id, expert_shard_id = mapping
if weight_name not in name: if weight_name not in name:
continue continue
is_expert_weight = True is_expert_weight = True
@@ -627,7 +629,7 @@ class FlashModel(nn.Module):
param, param,
loaded_weight, loaded_weight,
name_mapped, name_mapped,
shard_id=shard_id, shard_id=expert_shard_id,
expert_id=expert_id, expert_id=expert_id,
return_success=True, return_success=True,
) )
@@ -664,11 +666,14 @@ class FlashModel(nn.Module):
if isinstance(self.layers[layer_id], PPMissingLayer): if isinstance(self.layers[layer_id], PPMissingLayer):
continue continue
self_attn = self.layers[layer_id].self_attn[i] self_attn = self.layers[layer_id].self_attn[i]
if hasattr( if (
self.quant_config, "weight_block_size" self.quant_config is not None
) and self_attn.kv_b_proj.weight.dtype in ( and hasattr(self.quant_config, "weight_block_size")
torch.float8_e4m3fn, and self_attn.kv_b_proj.weight.dtype
torch.float8_e4m3fnuz, in (
torch.float8_e4m3fn,
torch.float8_e4m3fnuz,
)
): ):
weight_block_size = self.quant_config.weight_block_size weight_block_size = self.quant_config.weight_block_size
if weight_block_size is not None: if weight_block_size is not None:
@@ -266,11 +266,14 @@ class LongCatFlashMTP(nn.Module):
loaded_params.add(name) loaded_params.add(name)
spec_layer_id = self.config.num_hidden_layers * 2 spec_layer_id = self.config.num_hidden_layers * 2
self_attn = self.model.layers[str(spec_layer_id)].mtp_block.self_attn self_attn = self.model.layers[str(spec_layer_id)].mtp_block.self_attn
if hasattr( if (
self.quant_config, "weight_block_size" self.quant_config is not None
) and self_attn.kv_b_proj.weight.dtype in ( and hasattr(self.quant_config, "weight_block_size")
torch.float8_e4m3fn, and self_attn.kv_b_proj.weight.dtype
torch.float8_e4m3fnuz, in (
torch.float8_e4m3fn,
torch.float8_e4m3fnuz,
)
): ):
weight_block_size = self.quant_config.weight_block_size weight_block_size = self.quant_config.weight_block_size
if weight_block_size is not None: if weight_block_size is not None:
@@ -178,6 +178,7 @@ class FlashNgramModel(FlashModel):
if num_layers is not None and hf.num_hidden_layers != num_layers: if num_layers is not None and hf.num_hidden_layers != num_layers:
hf.num_hidden_layers = num_layers hf.num_hidden_layers = num_layers
super().__init__(vllm_config=vllm_config, prefix=prefix) super().__init__(vllm_config=vllm_config, prefix=prefix)
self.ngram_embeddings: NgramEmbedding | None
if get_pp_group().is_first_rank and uses_ngram_embedding(self.config): if get_pp_group().is_first_rank and uses_ngram_embedding(self.config):
self.ngram_embeddings = NgramEmbedding(self.config, self.embed_tokens) self.ngram_embeddings = NgramEmbedding(self.config, self.embed_tokens)
else: else:
+1 -1
View File
@@ -217,7 +217,7 @@ class Mistral3ProcessingInfo(BaseProcessingInfo):
def get_vision_encoder_info( def get_vision_encoder_info(
self, self,
mm_processor_kwargs: Mapping[str, object] | None = None, mm_processor_kwargs: Mapping[str, object] | None = None,
) -> Mistral3HFEncoderInfo: ) -> PixtralHFEncoderInfo:
processor = self.get_hf_processor() processor = self.get_hf_processor()
size = processor.image_processor.size size = processor.image_processor.size
merged_kwargs = self.ctx.get_merged_mm_kwargs(mm_processor_kwargs or {}) merged_kwargs = self.ctx.get_merged_mm_kwargs(mm_processor_kwargs or {})