From c615b1fd67f5b149c75aea592de8e563cb0c2da9 Mon Sep 17 00:00:00 2001 From: Taneem Ibrahim Date: Fri, 4 Sep 2026 06:50:43 -0500 Subject: [PATCH] [Mypy] Fix mypy typing for L models (#54177) Signed-off-by: Taneem Ibrahim --- tools/pre_commit/mypy.py | 1 - vllm/model_executor/models/laguna.py | 1 + vllm/model_executor/models/laguna_dflash.py | 26 +++---- vllm/model_executor/models/lfm2_vl.py | 21 ++++++ vllm/model_executor/models/lightonocr.py | 2 +- vllm/model_executor/models/llama.py | 17 ++++- vllm/model_executor/models/llama4.py | 8 +- vllm/model_executor/models/llama4_eagle.py | 26 ++++++- vllm/model_executor/models/llama_eagle.py | 27 +++++-- vllm/model_executor/models/llama_eagle3.py | 51 +++++++++++-- vllm/model_executor/models/llava.py | 16 ++-- vllm/model_executor/models/llava_next.py | 6 +- .../model_executor/models/llava_next_video.py | 4 +- vllm/model_executor/models/llava_onevision.py | 30 +++++--- .../model_executor/models/llava_onevision2.py | 73 ++++++++++++++----- vllm/model_executor/models/longcat_flash.py | 19 +++-- .../models/longcat_flash_mtp.py | 13 ++-- .../models/longcat_flash_ngram.py | 1 + vllm/model_executor/models/mistral3.py | 2 +- 19 files changed, 255 insertions(+), 89 deletions(-) diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index 55ebd6ef18..91c06a730a 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -113,7 +113,6 @@ SEPARATE_GROUPS = [ EXCLUDE = [ r"vllm/model_executor/models/[kK]", - r"vllm/model_executor/models/[lL]", r"vllm/model_executor/models/[nN]", r"vllm/model_executor/models/[oO]", r"vllm/model_executor/models/[pP]", diff --git a/vllm/model_executor/models/laguna.py b/vllm/model_executor/models/laguna.py index 115000588d..12046379fd 100644 --- a/vllm/model_executor/models/laguna.py +++ b/vllm/model_executor/models/laguna.py @@ -317,6 +317,7 @@ class LagunaAttention(nn.Module): # config.gating may be: # - True / "per-element": one gate per (head, head_dim) channel # - "per-head": one gate per head, broadcast across head_dim + self.g_proj: ColumnParallelLinear | None if self.gating: # v5 LagunaConfig uses ``gating=True`` for per-head; older configs # used ``"per-head"``. Accept both. ``"per-element"`` (or legacy diff --git a/vllm/model_executor/models/laguna_dflash.py b/vllm/model_executor/models/laguna_dflash.py index 28f299b717..3c46a4cbb5 100644 --- a/vllm/model_executor/models/laguna_dflash.py +++ b/vllm/model_executor/models/laguna_dflash.py @@ -69,7 +69,9 @@ class DFlashLagunaModel(DFlashQwen3Model, EagleModelMixin): prefix: str = "", ) -> None: nn.Module.__init__(self) - self.config = vllm_config.speculative_config.draft_model_config.hf_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + self.config = speculative_config.draft_model_config.hf_config self.vocab_size = self.config.vocab_size self.quant_config = get_draft_quant_config(vllm_config) @@ -216,24 +218,16 @@ class DFlashLagunaModel(DFlashQwen3Model, EagleModelMixin): return all_k_normed def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + if self.quant_config is not None: + weights = self.quant_config.get_cache_scale_mapper().apply(weights) params_dict = dict(self.named_parameters()) loaded_params: set[str] = set() for name, loaded_weight in weights: - if self.quant_config is not None and ( - scale_name := self.quant_config.get_cache_scale(name) - ): - param = params_dict[scale_name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - loaded_weight = ( - loaded_weight if loaded_weight.dim() == 0 else loaded_weight[0] - ) - weight_loader(param, loaded_weight) - loaded_params.add(scale_name) - continue if "scale" in name: - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: + remapped_name = maybe_remap_kv_scale_name(name, params_dict) + if remapped_name is None: continue + name = remapped_name param = params_dict[name] weight_loader = getattr(param, "weight_loader", default_weight_loader) weight_loader(param, loaded_weight) @@ -244,7 +238,9 @@ class DFlashLagunaModel(DFlashQwen3Model, EagleModelMixin): class DFlashLagunaForCausalLM(nn.Module, SupportsEagle3): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): nn.Module.__init__(self) - self.config = vllm_config.speculative_config.draft_model_config.hf_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + self.config = speculative_config.draft_model_config.hf_config if getattr(self.config, "draft_vocab_size", None) is None: raise ValueError("Laguna DFlash config requires `draft_vocab_size`.") self.has_own_embed_tokens = False diff --git a/vllm/model_executor/models/lfm2_vl.py b/vllm/model_executor/models/lfm2_vl.py index d64ee32f90..d27c36d828 100644 --- a/vllm/model_executor/models/lfm2_vl.py +++ b/vllm/model_executor/models/lfm2_vl.py @@ -3,6 +3,7 @@ import itertools import math +import typing from collections.abc import Iterable, Mapping, Sequence from typing import Annotated, Any, Literal @@ -17,6 +18,7 @@ from transformers.models.lfm2_vl.image_processing_lfm2_vl_fast import ( find_closest_aspect_ratio, round_by_factor, ) +from typing_extensions import Buffer from vllm.config import VllmConfig from vllm.config.multimodal import BaseDummyOptions @@ -229,6 +231,13 @@ class Lfm2VLProcessingInfo(BaseProcessingInfo): "max_image_tokens", image_processor.max_image_tokens ) tile_size = mm_kwargs.get("tile_size", image_processor.tile_size) + assert isinstance(downsample_factor, int) + assert isinstance(encoder_patch_size, int) + assert isinstance(max_pixels_tolerance, int | float) + assert isinstance(min_tiles, int) + assert isinstance(max_tiles, int) + assert isinstance(max_image_tokens, int) + assert isinstance(tile_size, int) do_image_splitting = not min_tiles == max_tiles == 1 is_image_large = self._is_image_too_large( @@ -337,6 +346,9 @@ class Lfm2VLProcessingInfo(BaseProcessingInfo): "encoder_patch_size", image_processor.encoder_patch_size ) tile_size = mm_kwargs.get("tile_size", image_processor.tile_size) + assert isinstance(downsample_factor, int) + assert isinstance(encoder_patch_size, int) + assert isinstance(tile_size, int) thumbnail_height_patches = int(spatial_shapes[-1][0].item()) thumbnail_width_patches = int(spatial_shapes[-1][1].item()) @@ -672,6 +684,7 @@ class Lfm2VLForConditionalGeneration( super().__init__() config: Lfm2VlConfig = vllm_config.model_config.hf_config multimodal_config = vllm_config.model_config.multimodal_config + assert multimodal_config is not None vision_config = config.vision_config quant_config = vllm_config.quant_config @@ -915,6 +928,14 @@ class Lfm2VLForConditionalGeneration( "min_image_tokens", getattr(self.config, "min_image_tokens", None) or 64, ) + if not isinstance( + value, + (str, Buffer, typing.SupportsInt, typing.SupportsIndex), + ): + raise TypeError( + "int() argument must be a string, a bytes-like object " + f"or a real number, not '{type(value).__name__}'" + ) return max(1, int(value)) def _get_lfm2vl_item_tile_slices( diff --git a/vllm/model_executor/models/lightonocr.py b/vllm/model_executor/models/lightonocr.py index 7f108ba556..0ce0d7f3d2 100644 --- a/vllm/model_executor/models/lightonocr.py +++ b/vllm/model_executor/models/lightonocr.py @@ -49,7 +49,7 @@ class LightOnOCRProcessingInfo(Mistral3ProcessingInfo): return PixtralHFEncoderInfo(self.get_hf_config()) -class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[LightOnOCRProcessingInfo]): +class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]): def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) diff --git a/vllm/model_executor/models/llama.py b/vllm/model_executor/models/llama.py index a595ee713e..7f2526f698 100644 --- a/vllm/model_executor/models/llama.py +++ b/vllm/model_executor/models/llama.py @@ -26,6 +26,7 @@ from collections.abc import Iterable from itertools import islice +from typing import TYPE_CHECKING import torch from torch import nn @@ -544,13 +545,25 @@ class LlamaForCausalLM( return loader.load_weights(weights) -class LlamaBidirectionalForSequenceClassification(as_seq_cls_model(LlamaForCausalLM)): +if TYPE_CHECKING: + _LlamaBidirectionalForSequenceClassificationBase = LlamaForCausalLM + _LlamaBidirectionalModelBase = LlamaForCausalLM +else: + _LlamaBidirectionalForSequenceClassificationBase = as_seq_cls_model( + LlamaForCausalLM + ) + _LlamaBidirectionalModelBase = as_embedding_model(LlamaForCausalLM) + + +class LlamaBidirectionalForSequenceClassification( + _LlamaBidirectionalForSequenceClassificationBase +): # This class sets the correct attention type and pooling type # through LlamaBidirectionalConfig. pass -class LlamaBidirectionalModel(as_embedding_model(LlamaForCausalLM)): +class LlamaBidirectionalModel(_LlamaBidirectionalModelBase): # This class sets the correct attention type and pooling type # through LlamaBidirectionalConfig. pass diff --git a/vllm/model_executor/models/llama4.py b/vllm/model_executor/models/llama4.py index 2a073d2dc4..3dc0e9a904 100644 --- a/vllm/model_executor/models/llama4.py +++ b/vllm/model_executor/models/llama4.py @@ -18,7 +18,7 @@ # limitations under the License. """Inference-only LLaMA model compatible with HuggingFace weights.""" -from collections.abc import Iterable +from collections.abc import Callable, Iterable import torch from torch import nn @@ -576,6 +576,7 @@ class Llama4Model(LlamaModel): params_dict = dict(self.named_parameters()) # The module parameters that have been loaded. loaded_params: set[str] = set() + weight_loader: Callable[..., object] # Iterate over all the weights and load them into module parameters. for name, loaded_weight in weights: @@ -612,9 +613,10 @@ class Llama4Model(LlamaModel): # quant config's `get_cache_scale_mapper` does not cover # (idempotent for names already renamed by the mapper). if name.endswith("scale"): - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: + remapped_name = maybe_remap_kv_scale_name(name, params_dict) + if remapped_name is None: continue + name = remapped_name # Load the weight into the module parameter with corresponding # shard id and exit the for loop and the else block. diff --git a/vllm/model_executor/models/llama4_eagle.py b/vllm/model_executor/models/llama4_eagle.py index ba9a8e3254..975ee28dc2 100644 --- a/vllm/model_executor/models/llama4_eagle.py +++ b/vllm/model_executor/models/llama4_eagle.py @@ -17,6 +17,7 @@ # limitations under the License. from collections.abc import Iterable +from typing import TYPE_CHECKING import torch import torch.nn as nn @@ -46,6 +47,19 @@ from .utils import ( logger = init_logger(__name__) +if TYPE_CHECKING: + + class _EagleLlama4ForCausalLMBase(nn.Module): + def set_moe_parameters(self) -> None: ... + + def permute_qk_weight_for_rotary( + self, name: str, loaded_weight: torch.Tensor + ) -> tuple[str, torch.Tensor]: ... + +else: + _EagleLlama4ForCausalLMBase = Llama4ForCausalLM + + @support_torch_compile class LlamaModel(nn.Module): hf_to_vllm_mapper = WeightsMapper( @@ -68,7 +82,9 @@ class LlamaModel(nn.Module): quant_config: QuantizationConfig | None = None, ) -> None: super().__init__() - self.config = vllm_config.speculative_config.draft_model_config.hf_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + self.config = speculative_config.draft_model_config.hf_config self.validate_and_update_config(start_layer_id, quant_config) self.vocab_size = self.config.vocab_size self.embed_tokens = VocabParallelEmbedding( @@ -152,16 +168,18 @@ class LlamaModel(nn.Module): } -class EagleLlama4ForCausalLM(Llama4ForCausalLM, SupportsMultiModalEmbeddings): +class EagleLlama4ForCausalLM(_EagleLlama4ForCausalLMBase, SupportsMultiModalEmbeddings): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): nn.Module.__init__(self) - self.config = vllm_config.speculative_config.draft_model_config.hf_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + self.config = speculative_config.draft_model_config.hf_config target_layer_num = vllm_config.model_config.get_num_layers( vllm_config.parallel_config ) # draft model quantization config may differ from target model quant_config = VllmConfig.get_quantization_config( - vllm_config.speculative_config.draft_model_config, vllm_config.load_config + speculative_config.draft_model_config, vllm_config.load_config ) self.model = LlamaModel( vllm_config=vllm_config, diff --git a/vllm/model_executor/models/llama_eagle.py b/vllm/model_executor/models/llama_eagle.py index 50706bb326..563609415f 100644 --- a/vllm/model_executor/models/llama_eagle.py +++ b/vllm/model_executor/models/llama_eagle.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections.abc import Iterable +from typing import TYPE_CHECKING import torch import torch.nn as nn @@ -14,7 +15,10 @@ from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization.base_config import QuantizationConfig from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding -from vllm.model_executor.models.llama import LlamaDecoderLayer, LlamaForCausalLM +from vllm.model_executor.models.llama import ( + LlamaDecoderLayer as BaseLlamaDecoderLayer, +) +from vllm.model_executor.models.llama import LlamaForCausalLM from .utils import ( AutoWeightsLoader, @@ -27,7 +31,16 @@ from .utils import ( logger = init_logger(__name__) -class LlamaDecoderLayer(LlamaDecoderLayer): +if TYPE_CHECKING: + + class _EagleLlamaForCausalLMBase(nn.Module): + pass + +else: + _EagleLlamaForCausalLMBase = LlamaForCausalLM + + +class LlamaDecoderLayer(BaseLlamaDecoderLayer): def __init__( self, vllm_config: VllmConfig, @@ -69,7 +82,9 @@ class LlamaModel(nn.Module): start_layer_id: int = 0, ) -> None: super().__init__() - self.config = vllm_config.speculative_config.draft_model_config.hf_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + self.config = speculative_config.draft_model_config.hf_config self.vocab_size = self.config.vocab_size # Get drafter's quantization config @@ -128,10 +143,12 @@ class LlamaModel(nn.Module): return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) -class EagleLlamaForCausalLM(LlamaForCausalLM): +class EagleLlamaForCausalLM(_EagleLlamaForCausalLMBase): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): nn.Module.__init__(self) - self.config = vllm_config.speculative_config.draft_model_config.hf_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + self.config = speculative_config.draft_model_config.hf_config # Ensure draft_vocab_size is set # default to the base vocab size when absent if getattr(self.config, "draft_vocab_size", None) is None: diff --git a/vllm/model_executor/models/llama_eagle3.py b/vllm/model_executor/models/llama_eagle3.py index 93c94e7599..fab611cf31 100644 --- a/vllm/model_executor/models/llama_eagle3.py +++ b/vllm/model_executor/models/llama_eagle3.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections.abc import Iterable +from typing import TYPE_CHECKING import torch import torch.nn as nn @@ -18,7 +19,14 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.models.llama import LlamaDecoderLayer, LlamaForCausalLM +from vllm.model_executor.models.llama import ( + LlamaAttention, + LlamaForCausalLM, + LlamaMLP, +) +from vllm.model_executor.models.llama import ( + LlamaDecoderLayer as BaseLlamaDecoderLayer, +) from vllm.multimodal.inputs import NestedTensors from .utils import ( @@ -32,7 +40,31 @@ from .utils import ( logger = init_logger(__name__) -class LlamaDecoderLayer(LlamaDecoderLayer): +if TYPE_CHECKING: + + class _Eagle3LlamaDecoderLayerBase(nn.Module): + hidden_size: int + self_attn: LlamaAttention + mlp: LlamaMLP + input_layernorm: RMSNorm + post_attention_layernorm: RMSNorm + + def __init__( + self, + vllm_config: VllmConfig, + prefix: str = "", + config: LlamaConfig | None = None, + ) -> None: ... + + class _Eagle3LlamaForCausalLMBase(nn.Module): + pass + +else: + _Eagle3LlamaDecoderLayerBase = BaseLlamaDecoderLayer + _Eagle3LlamaForCausalLMBase = LlamaForCausalLM + + +class LlamaDecoderLayer(_Eagle3LlamaDecoderLayerBase): def __init__( self, vllm_config: VllmConfig, @@ -136,7 +168,9 @@ class LlamaModel(nn.Module): prefix: str = "", ) -> None: super().__init__() - self.config = vllm_config.speculative_config.draft_model_config.hf_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + self.config = speculative_config.draft_model_config.hf_config self.vocab_size = self.config.vocab_size # Get drafter's quantization config @@ -187,7 +221,7 @@ class LlamaModel(nn.Module): self.fc_input_size = target_hidden_size * self.num_aux_hidden_states if self.norm_before_fc: - self.input_norm = RMSNorm( + self.input_norm: RMSNorm | None = RMSNorm( self.fc_input_size, eps=self.config.rms_norm_eps, ) @@ -269,10 +303,12 @@ class LlamaModel(nn.Module): return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) -class Eagle3LlamaForCausalLM(LlamaForCausalLM): +class Eagle3LlamaForCausalLM(_Eagle3LlamaForCausalLMBase): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): nn.Module.__init__(self) - self.config = vllm_config.speculative_config.draft_model_config.hf_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + self.config = speculative_config.draft_model_config.hf_config # Ensure draft_vocab_size is set # default to the base vocab size when absent if getattr(self.config, "draft_vocab_size", None) is None: @@ -306,7 +342,7 @@ class Eagle3LlamaForCausalLM(LlamaForCausalLM): requires_grad=False, ) - self.use_parallel_drafting = vllm_config.speculative_config.parallel_drafting + self.use_parallel_drafting = speculative_config.parallel_drafting if self.use_parallel_drafting: self.register_buffer( @@ -365,6 +401,7 @@ class Eagle3LlamaForCausalLM(LlamaForCausalLM): # combine multiple auxiliary hidden states returned by eagle3 if self.model.norm_before_fc: + assert self.model.input_norm is not None hidden_states = self.model.input_norm(hidden_states) # `norm_before_fc` adds a single RMSNorm before the FC layer, whereas `fc_norm` diff --git a/vllm/model_executor/models/llava.py b/vllm/model_executor/models/llava.py index c3ec7a4daa..1f1f45bfea 100644 --- a/vllm/model_executor/models/llava.py +++ b/vllm/model_executor/models/llava.py @@ -3,7 +3,7 @@ from abc import abstractmethod from collections.abc import Iterable, Mapping, Sequence -from typing import Annotated, Final, Literal, Protocol, TypeAlias, TypeVar +from typing import Annotated, Literal, Protocol, TypeAlias, TypeVar import torch import torch.nn as nn @@ -159,14 +159,14 @@ class LlavaMultiModalProjector(nn.Module): class LlavaLikeConfig(Protocol): - vision_config: Final[PretrainedConfig] - image_token_index: Final[int] - vision_feature_select_strategy: Final[str] - vision_feature_layer: Final[int | list[int]] + vision_config: PretrainedConfig + image_token_index: int + vision_feature_select_strategy: str + vision_feature_layer: int | list[int] class LlavaLikeProcessor(Protocol): - image_token: Final[str] + image_token: str class BaseLlavaProcessingInfo(BaseProcessingInfo): @@ -286,6 +286,7 @@ class BaseLlavaMultiModalProcessor(BaseMultiModalProcessor[_I]): if isinstance(images, ImageEmbeddingItems): num_image_tokens = images.get_feature_size(item_idx) else: + assert isinstance(images, ImageProcessorItems) image_size = images.get_image_size(item_idx) num_image_tokens = self.info.get_num_image_tokens( image_width=image_size.width, @@ -372,6 +373,7 @@ class PixtralHFMultiModalProcessor(BaseMultiModalProcessor[PixtralHFProcessingIn image_end_id = vocab[processor.image_end_token] assert isinstance(hf_config.vision_config, PixtralVisionConfig) + assert isinstance(hf_config, LlavaConfig) encoder_info = PixtralHFEncoderInfo(hf_config) def get_replacement(item_idx: int): @@ -642,7 +644,7 @@ class LlavaForConditionalGeneration( self, image_input: LlavaImageInputs, ) -> torch.Tensor | tuple[torch.Tensor, ...]: - if image_input["type"] == "image_embeds": + if isinstance(image_input, LlavaImageEmbeddingInputs): return image_input["data"] image_features = self._process_image_pixels(image_input) diff --git a/vllm/model_executor/models/llava_next.py b/vllm/model_executor/models/llava_next.py index cf93da2924..fee9a4ac59 100644 --- a/vllm/model_executor/models/llava_next.py +++ b/vllm/model_executor/models/llava_next.py @@ -3,7 +3,7 @@ from abc import abstractmethod from collections.abc import Iterable, Mapping -from typing import Annotated, Final, Literal, Protocol, TypeAlias, TypeVar +from typing import Annotated, Literal, Protocol, TypeAlias, TypeVar import torch import torch.nn as nn @@ -88,7 +88,7 @@ LlavaNextImageInputs: TypeAlias = ( class LlavaNextLikeConfig(LlavaLikeConfig, Protocol): - image_grid_pinpoints: Final[list[list[int]]] + image_grid_pinpoints: list[list[int]] class LlavaNextProcessingInfo(BaseLlavaProcessingInfo): @@ -483,7 +483,7 @@ class LlavaNextForConditionalGeneration( self, image_input: LlavaNextImageInputs, ) -> torch.Tensor | list[torch.Tensor]: - if image_input["type"] == "image_embeds": + if isinstance(image_input, LlavaNextImageEmbeddingInputs): return image_input["data"] patch_embeddings = self._process_image_pixels(image_input) diff --git a/vllm/model_executor/models/llava_next_video.py b/vllm/model_executor/models/llava_next_video.py index 7794183734..038da17b75 100644 --- a/vllm/model_executor/models/llava_next_video.py +++ b/vllm/model_executor/models/llava_next_video.py @@ -10,7 +10,7 @@ import torch.nn as nn from transformers import BatchFeature, LlavaNextVideoConfig, LlavaNextVideoProcessor from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.models.clip import CLIPVisionModel @@ -181,6 +181,7 @@ class LlavaNextVideoDummyInputsBuilder( ) video_overrides = mm_options.get("video") + assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions) return { "video": self._get_dummy_videos( @@ -220,6 +221,7 @@ class LlavaNextVideoMultiModalProcessor( if isinstance(videos, VideoEmbeddingItems): num_video_tokens = videos.get_feature_size(item_idx) else: + assert isinstance(videos, VideoProcessorItems) image_size = videos.get_frame_size(item_idx) num_video_tokens = self.info.get_num_video_tokens( image_width=image_size.width, diff --git a/vllm/model_executor/models/llava_onevision.py b/vllm/model_executor/models/llava_onevision.py index 848372f1f6..631f73e64d 100644 --- a/vllm/model_executor/models/llava_onevision.py +++ b/vllm/model_executor/models/llava_onevision.py @@ -3,7 +3,7 @@ import math from collections.abc import Iterable, Mapping, Sequence -from typing import Annotated, Final, Literal, Protocol, TypeAlias +from typing import Annotated, Literal, Protocol, TypeAlias, TypedDict import torch import torch.nn as nn @@ -14,7 +14,7 @@ from transformers.models.llava_onevision.modeling_llava_onevision import ( ) from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn from vllm.multimodal import MULTIMODAL_REGISTRY @@ -123,7 +123,12 @@ LlavaOnevisionMultiInputs: TypeAlias = ( class LlavaOnevisionLikeConfig(LlavaNextLikeConfig, Protocol): - video_token_index: Final[int] + video_token_index: int + + +class LlavaOnevisionInputsByModality(TypedDict, total=False): + image: LlavaOnevisionImageInputs | None + video: LlavaOnevisionVideoPixelInputs | None class LlavaOnevisionProcessingInfo(LlavaNextProcessingInfo): @@ -288,6 +293,7 @@ class LlavaOnevisionDummyInputsBuilder( image_overrides = mm_options.get("image") video_overrides = mm_options.get("video") + assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions) return { "image": self._get_dummy_images( @@ -417,6 +423,7 @@ class LlavaOnevisionMultiModalProcessor( if isinstance(videos, VideoEmbeddingItems): num_video_tokens = videos.get_feature_size(item_idx) else: + assert isinstance(videos, VideoProcessorItems) image_size = videos.get_frame_size(item_idx) num_video_tokens = self.info.get_num_video_tokens( image_width=image_size.width, @@ -570,8 +577,10 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp }, ) - def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: - mm_input_by_modality = {} + def _parse_and_validate_multimodal_inputs( + self, **kwargs: object + ) -> LlavaOnevisionInputsByModality: + mm_input_by_modality = LlavaOnevisionInputsByModality() # Preserve the order of modalities if there are multiple of them # from the order of kwargs. @@ -749,7 +758,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp self, image_input: LlavaOnevisionImageInputs, ) -> torch.Tensor | list[torch.Tensor]: - if image_input["type"] == "image_embeds": + if isinstance(image_input, LlavaOnevisionImageEmbeddingInputs): return image_input["data"] patch_embeddings = self._process_image_pixels(image_input) @@ -860,12 +869,15 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp # NOTE: It is important to iterate over the keys in this dictionary # to preserve the order of the modalities. for modality in mm_input_by_modality: - multimodal_input = mm_input_by_modality[modality] if modality == "image": - image_embeddings = self._process_image_input(multimodal_input) + image_input = mm_input_by_modality["image"] + assert image_input is not None + image_embeddings = self._process_image_input(image_input) multimodal_embeddings += tuple(image_embeddings) if modality == "video": - video_embeddings = self._process_video_pixels(multimodal_input) + video_input = mm_input_by_modality["video"] + assert video_input is not None + video_embeddings = self._process_video_pixels(video_input) multimodal_embeddings += tuple(video_embeddings) return multimodal_embeddings diff --git a/vllm/model_executor/models/llava_onevision2.py b/vllm/model_executor/models/llava_onevision2.py index ba3678fbc8..7eda9dc320 100644 --- a/vllm/model_executor/models/llava_onevision2.py +++ b/vllm/model_executor/models/llava_onevision2.py @@ -24,12 +24,15 @@ from __future__ import annotations import hashlib import importlib -from collections.abc import Callable, Iterable, Mapping, Sequence +from collections.abc import Callable, Iterable, Mapping, Sequence, Sized from functools import lru_cache from typing import ( Annotated, Any, Literal, + SupportsIndex, + SupportsInt, + TypedDict, ) import numpy as np @@ -45,6 +48,7 @@ from transformers.dynamic_module_utils import ( ) from transformers.models.qwen2_vl import Qwen2VLImageProcessor from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize +from typing_extensions import Buffer from vllm.compilation.decorators import ( should_torch_compile_mm_encoder, @@ -730,6 +734,11 @@ LlavaOnevision2ImageInputs = ( ) +class LlavaOnevision2InputsByModality(TypedDict, total=False): + images: LlavaOnevision2ImageInputs | None + videos: LlavaOnevision2VideoPixelInputs | None + + class LlavaOnevision2VisionRotaryEmbedding(nn.Module): """3-D rotary frequency constructor with 4:6:6 (T:H:W) split. @@ -1468,7 +1477,7 @@ class LlavaOnevision2DummyInputsBuilder( n_img = mm_counts.get("image", 0) n_vid = mm_counts.get("video", 0) w, h = self.info.get_image_size_with_most_features() - out: MultiModalDataDict = {} + out: dict[str, object] = {} if n_img: out["image"] = self._get_dummy_images(width=w, height=h, num_images=n_img) if n_vid: @@ -1586,6 +1595,8 @@ class LlavaOnevision2MultiModalProcessor( # array, or tensor, and ``and `` would raise on the ambiguous # truth value of a multi-element array. _videos = mm_data.get("videos") + if _videos is not None and not isinstance(_videos, Sized): + raise TypeError(f"object of type '{type(_videos).__name__}' has no len()") videos_present = _videos is not None and len(_videos) > 0 codec_video_paths = ( @@ -1666,15 +1677,24 @@ class LlavaOnevision2MultiModalProcessor( # provided by the backend's ``compute_frames_index_to_sample``; SSRF / # local-file gating is enforced by the connector before decoding. if videos_present: - timestamp_decimals = int( - hf_processor_mm_kwargs.get( - "timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS - ) + timestamp_decimals = hf_processor_mm_kwargs.get( + "timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS ) + if not isinstance( + timestamp_decimals, (str, Buffer, SupportsInt, SupportsIndex) + ): + raise TypeError( + "int() argument must be a string, a bytes-like object " + f"or a real number, not '{type(timestamp_decimals).__name__}'" + ) + timestamp_decimals = int(timestamp_decimals) per_video_frames: list[list[Image.Image]] = [] per_video_timestamps: list[list[float]] = [] - for item in mm_data["videos"]: + videos = mm_data["videos"] + if not isinstance(videos, Iterable): + raise TypeError(f"Unsupported videos input: {type(videos)}") + for item in videos: pil_frames, timestamps = _frame_video_to_pil_and_timestamps(item) per_video_frames.append(pil_frames) per_video_timestamps.append(timestamps) @@ -1902,18 +1922,21 @@ class LlavaOnevision2MultiModalProcessor( def get_image_replacement(item_idx: int): out_item = out_mm_kwargs["image"][item_idx] grid_thw = out_item["image_grid_thw"].data + assert isinstance(grid_thw, torch.Tensor) n = int(grid_thw.prod(-1).sum()) // merge_length return [image_pad_id] * n def get_video_replacement(item_idx: int): out_item = out_mm_kwargs["video"][item_idx] grid_thw = out_item["video_grid_thw"].data + assert isinstance(grid_thw, torch.Tensor) is_codec_field = out_item.get("video_is_codec") - is_codec = ( - bool(int(is_codec_field.data.item())) - if is_codec_field is not None - else False - ) + if is_codec_field is not None: + is_codec_data = is_codec_field.data + assert isinstance(is_codec_data, torch.Tensor) + is_codec = bool(int(is_codec_data.item())) + else: + is_codec = False tokens: list[int] = [] if is_codec: @@ -1924,6 +1947,8 @@ class LlavaOnevision2MultiModalProcessor( # run, emit ``<|vision_start|><|vision_end|>\n``. patch_positions = out_item["patch_positions_videos"].data fps_t = out_item["codec_fps"].data + assert isinstance(patch_positions, torch.Tensor) + assert isinstance(fps_t, torch.Tensor) fps = float(int(fps_t.item())) / 1_000_000_000.0 runs = _codec_timestamp_runs( patch_positions, fps, image_processor.merge_size @@ -1938,6 +1963,7 @@ class LlavaOnevision2MultiModalProcessor( tokens.extend(newline_ids) else: timestamps = out_item["frame_timestamps"].data + assert isinstance(timestamps, torch.Tensor) T_total = int(grid_thw.shape[0]) for t in range(T_total): sec = float(timestamps[t].item()) @@ -2121,7 +2147,7 @@ class LlavaOnevision2ForConditionalGeneration( grid_thw = image_input["image_grid_thw"] assert grid_thw.ndim == 2 - if image_input["type"] == "image_embeds": + if isinstance(image_input, LlavaOnevision2ImageEmbeddingInputs): image_embeds = image_input["image_embeds"] else: image_embeds = self.visual( @@ -2166,8 +2192,13 @@ class LlavaOnevision2ForConditionalGeneration( ) if isinstance(video_num_frames, list): video_num_frames = torch.cat([v.flatten() for v in video_num_frames]) - else: + elif isinstance(video_num_frames, torch.Tensor): video_num_frames = video_num_frames.flatten() + else: + raise TypeError( + "video_num_frames must be a tensor or a list of tensors, " + f"got {type(video_num_frames)}" + ) return LlavaOnevision2VideoPixelInputs( type="pixel_values_videos", pixel_values_videos=pixel_values_videos, @@ -2202,8 +2233,10 @@ class LlavaOnevision2ForConditionalGeneration( cursor += n return video_embeds.split(sizes) - def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: - modalities = {} + def _parse_and_validate_multimodal_inputs( + self, **kwargs: object + ) -> LlavaOnevision2InputsByModality: + modalities = LlavaOnevision2InputsByModality() for key in kwargs: if key in ("pixel_values", "image_embeds") and "images" not in modalities: modalities["images"] = self._parse_and_validate_image_input(**kwargs) @@ -2221,9 +2254,13 @@ class LlavaOnevision2ForConditionalGeneration( multimodal_embeddings: tuple[torch.Tensor, ...] = () for modality in modalities: if modality == "images": - multimodal_embeddings += self._process_image_input(modalities["images"]) + image_input = modalities["images"] + assert image_input is not None + multimodal_embeddings += self._process_image_input(image_input) elif modality == "videos": - multimodal_embeddings += self._process_video_input(modalities["videos"]) + video_input = modalities["videos"] + assert video_input is not None + multimodal_embeddings += self._process_video_input(video_input) return multimodal_embeddings def get_input_embeddings( diff --git a/vllm/model_executor/models/longcat_flash.py b/vllm/model_executor/models/longcat_flash.py index 2c6dedae35..4178d63ff4 100644 --- a/vllm/model_executor/models/longcat_flash.py +++ b/vllm/model_executor/models/longcat_flash.py @@ -83,6 +83,8 @@ logger = init_logger(__name__) class FlashConfig(PretrainedConfig): """Flash model configuration.""" + moe_intermediate_size: int + model_type = "longcat_flash" keys_to_ignore_at_inference = ["past_key_values"] @@ -604,7 +606,7 @@ class FlashModel(nn.Module): else: is_expert_weight = False for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping + param_name, weight_name, expert_id, expert_shard_id = mapping if weight_name not in name: continue is_expert_weight = True @@ -627,7 +629,7 @@ class FlashModel(nn.Module): param, loaded_weight, name_mapped, - shard_id=shard_id, + shard_id=expert_shard_id, expert_id=expert_id, return_success=True, ) @@ -664,11 +666,14 @@ class FlashModel(nn.Module): if isinstance(self.layers[layer_id], PPMissingLayer): continue self_attn = self.layers[layer_id].self_attn[i] - if hasattr( - self.quant_config, "weight_block_size" - ) and self_attn.kv_b_proj.weight.dtype in ( - torch.float8_e4m3fn, - torch.float8_e4m3fnuz, + if ( + self.quant_config is not None + and hasattr(self.quant_config, "weight_block_size") + and self_attn.kv_b_proj.weight.dtype + in ( + torch.float8_e4m3fn, + torch.float8_e4m3fnuz, + ) ): weight_block_size = self.quant_config.weight_block_size if weight_block_size is not None: diff --git a/vllm/model_executor/models/longcat_flash_mtp.py b/vllm/model_executor/models/longcat_flash_mtp.py index 26fff05da6..360eb9c763 100644 --- a/vllm/model_executor/models/longcat_flash_mtp.py +++ b/vllm/model_executor/models/longcat_flash_mtp.py @@ -266,11 +266,14 @@ class LongCatFlashMTP(nn.Module): loaded_params.add(name) spec_layer_id = self.config.num_hidden_layers * 2 self_attn = self.model.layers[str(spec_layer_id)].mtp_block.self_attn - if hasattr( - self.quant_config, "weight_block_size" - ) and self_attn.kv_b_proj.weight.dtype in ( - torch.float8_e4m3fn, - torch.float8_e4m3fnuz, + if ( + self.quant_config is not None + and hasattr(self.quant_config, "weight_block_size") + and self_attn.kv_b_proj.weight.dtype + in ( + torch.float8_e4m3fn, + torch.float8_e4m3fnuz, + ) ): weight_block_size = self.quant_config.weight_block_size if weight_block_size is not None: diff --git a/vllm/model_executor/models/longcat_flash_ngram.py b/vllm/model_executor/models/longcat_flash_ngram.py index e01672bb14..76aa80013e 100644 --- a/vllm/model_executor/models/longcat_flash_ngram.py +++ b/vllm/model_executor/models/longcat_flash_ngram.py @@ -178,6 +178,7 @@ class FlashNgramModel(FlashModel): if num_layers is not None and hf.num_hidden_layers != num_layers: hf.num_hidden_layers = num_layers super().__init__(vllm_config=vllm_config, prefix=prefix) + self.ngram_embeddings: NgramEmbedding | None if get_pp_group().is_first_rank and uses_ngram_embedding(self.config): self.ngram_embeddings = NgramEmbedding(self.config, self.embed_tokens) else: diff --git a/vllm/model_executor/models/mistral3.py b/vllm/model_executor/models/mistral3.py index 3463e59e76..79c53b1c7e 100644 --- a/vllm/model_executor/models/mistral3.py +++ b/vllm/model_executor/models/mistral3.py @@ -217,7 +217,7 @@ class Mistral3ProcessingInfo(BaseProcessingInfo): def get_vision_encoder_info( self, mm_processor_kwargs: Mapping[str, object] | None = None, - ) -> Mistral3HFEncoderInfo: + ) -> PixtralHFEncoderInfo: processor = self.get_hf_processor() size = processor.image_processor.size merged_kwargs = self.ctx.get_merged_mm_kwargs(mm_processor_kwargs or {})