diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index b5cbc9e136..ab220da8a3 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -112,8 +112,6 @@ SEPARATE_GROUPS = [ ] EXCLUDE = [ - r"vllm/model_executor/models/[eE]", - r"vllm/model_executor/models/[fF]", r"vllm/model_executor/models/[gG]", r"vllm/model_executor/models/[hH]", r"vllm/model_executor/models/[iI]", diff --git a/vllm/compilation/decorators.py b/vllm/compilation/decorators.py index ad710a6fe3..ceb3ad3170 100644 --- a/vllm/compilation/decorators.py +++ b/vllm/compilation/decorators.py @@ -106,8 +106,10 @@ def support_torch_compile( @overload def support_torch_compile( *, - dynamic_arg_dims: dict[str, int | list[int] | dict[int, str]] | None, - mark_unbacked_dims: dict[str, int | list[int]] | None, + dynamic_arg_dims: dict[str, int | list[int] | dict[int, str]] | None = None, + mark_unbacked_dims: dict[str, int | list[int]] | None = None, + enable_if: Callable[[VllmConfig], bool] | None = None, + is_encoder: bool = False, ) -> Callable[[type[_T]], type[_T]]: ... diff --git a/vllm/model_executor/models/eagle2_5_vl.py b/vllm/model_executor/models/eagle2_5_vl.py index 30b8173f19..8fdd7098f2 100644 --- a/vllm/model_executor/models/eagle2_5_vl.py +++ b/vllm/model_executor/models/eagle2_5_vl.py @@ -150,6 +150,7 @@ class Eagle2_5_VLForConditionalGeneration( config = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config multimodal_config = vllm_config.model_config.multimodal_config + assert multimodal_config is not None self.config = config self.multimodal_config = multimodal_config @@ -187,7 +188,7 @@ class Eagle2_5_VLForConditionalGeneration( prefix=maybe_prefix(prefix, "language_model"), ) - self.img_context_token_id = None + self.img_context_token_id: int | None = None self.make_empty_intermediate_tensors = ( self.language_model.make_empty_intermediate_tensors diff --git a/vllm/model_executor/models/ernie45_moe.py b/vllm/model_executor/models/ernie45_moe.py index 746d12c596..2683e94f51 100644 --- a/vllm/model_executor/models/ernie45_moe.py +++ b/vllm/model_executor/models/ernie45_moe.py @@ -127,7 +127,7 @@ class Ernie4_5_MoeMoE(nn.Module): self.layer_idx = layer_idx self.tp_size = get_tensor_model_parallel_world_size() - self.moe_num_shared_experts = getattr(config, "moe_num_shared_experts", None) + self.moe_num_shared_experts = getattr(config, "moe_num_shared_experts", 0) self.ep_group = get_ep_group().device_group self.ep_size = self.ep_group.size() self.n_routed_experts: int = config.moe_num_experts @@ -163,6 +163,7 @@ class Ernie4_5_MoeMoE(nn.Module): torch.empty(config.moe_num_experts, dtype=torch.float32) ) + self.shared_experts: Ernie4_5_MoeMLP | None if self.has_shared_experts: intermediate_size = ( config.moe_intermediate_size * config.moe_num_shared_experts diff --git a/vllm/model_executor/models/ernie45_vl.py b/vllm/model_executor/models/ernie45_vl.py index a1de2ae54e..96466bad09 100644 --- a/vllm/model_executor/models/ernie45_vl.py +++ b/vllm/model_executor/models/ernie45_vl.py @@ -1076,7 +1076,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin def _pixel_values_norm( self, pixel_values: torch.Tensor, - mm_kwargs: object, + mm_kwargs: Mapping[str, object], ) -> torch.Tensor: hf_config = self.info.get_hf_config() vision_config = hf_config.vision_config @@ -1125,6 +1125,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) + mm_data = dict(mm_data) if "images" not in mm_data: mm_data["images"] = [] if "videos" not in mm_data: @@ -1136,15 +1137,15 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin hf_processor, "supports_video_metadata", False ) - if mm_data["videos"] and not supports_video_metadata: + videos = mm_data["videos"] + assert isinstance(videos, Sequence) + if videos and not supports_video_metadata: # Old HF processor, unwrap tuple to pure frames logger.warning_once( "HF processor doesn't support video metadata. " "Timestamps will NOT be rendered. Please upgrade the model." ) - mm_data["videos"] = [ - v[0] if isinstance(v, tuple) else v for v in mm_data["videos"] - ] + mm_data["videos"] = [v[0] if isinstance(v, tuple) else v for v in videos] processor_output = self.info.ctx.call_hf_processor( hf_processor, @@ -1286,6 +1287,7 @@ class Ernie4_5_VLDummyInputsBuilder(BaseDummyInputsBuilder[Ernie4_5_VLProcessing image_overrides = mm_options.get("image") video_overrides = mm_options.get("video") + assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions) return { "image": self._get_dummy_images( @@ -1541,10 +1543,14 @@ class Ernie4_5_VLMoeForConditionalGeneration( offset = mm_feature.mm_position.offset if mm_feature.modality == "image": - t, h, w = mm_feature.data["image_grid_thw"].data.tolist() + grid_thw = mm_feature.data["image_grid_thw"].data + assert isinstance(grid_thw, torch.Tensor) + t, h, w = grid_thw.tolist() yield offset, t, h // spatial_conv_size, w // spatial_conv_size elif mm_feature.modality == "video": - t, h, w = mm_feature.data["video_grid_thw"].data.tolist() + grid_thw = mm_feature.data["video_grid_thw"].data + assert isinstance(grid_thw, torch.Tensor) + t, h, w = grid_thw.tolist() yield ( offset, t // temporal_conv_size, @@ -1563,12 +1569,11 @@ class Ernie4_5_VLMoeForConditionalGeneration( if pixel_values is None: return None - if pixel_values is not None: - return Ernie4_5_VLImagePixelInputs( - type="pixel_values", - pixel_values=pixel_values, - image_grid_thw=image_grid_thw, - ) + return Ernie4_5_VLImagePixelInputs( + type="pixel_values", + pixel_values=pixel_values, + image_grid_thw=image_grid_thw, + ) def _parse_and_validate_video_input( self, **kwargs: object @@ -1579,12 +1584,11 @@ class Ernie4_5_VLMoeForConditionalGeneration( if pixel_values_videos is None: return None - if pixel_values_videos is not None: - return Ernie4_5_VLVideoPixelInputs( - type="pixel_values_videos", - pixel_values_videos=pixel_values_videos, - video_grid_thw=video_grid_thw, - ) + return Ernie4_5_VLVideoPixelInputs( + type="pixel_values_videos", + pixel_values_videos=pixel_values_videos, + video_grid_thw=video_grid_thw, + ) # -- SupportsEncoderCudaGraph protocol methods -- # @@ -1748,6 +1752,7 @@ class Ernie4_5_VLMoeForConditionalGeneration( # the actual batch grid_thw, then scatter the post-merge embeddings. # Ernie only uses the single "default" encoder path. output = outputs["default"] + assert batch_mm_kwargs is not None grid_thw_cpu = batch_mm_kwargs["image_grid_thw"] grid_thw = grid_thw_cpu.to(output.device, non_blocking=True) # The valid token count slices the graph output for the eager @@ -1799,7 +1804,9 @@ class Ernie4_5_VLMoeForConditionalGeneration( return video_embeds.split(sizes.tolist()) def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: - modalities = {} + modalities: dict[ + str, Ernie4_5_VLImageInputs | Ernie4_5_VLVideoInputs | None + ] = {} # Preserve the order of modalities if there are multiple of them # from the order of kwargs. diff --git a/vllm/model_executor/models/ernie45_vl_moe.py b/vllm/model_executor/models/ernie45_vl_moe.py index e18a3fec50..ccc4b9b868 100644 --- a/vllm/model_executor/models/ernie45_vl_moe.py +++ b/vllm/model_executor/models/ernie45_vl_moe.py @@ -232,6 +232,7 @@ class Ernie4_5_VLMoeMoE(nn.Module): assert text_moe_layer_start_index <= text_moe_layer_end_index + self.shared_experts: Ernie4_5_VLMoeMLP | None if self.has_shared_experts: intermediate_size = ( config.moe_intermediate_size[0] * config.moe_num_shared_experts @@ -771,9 +772,10 @@ class Ernie4_5_VLMoeForCausalLM(nn.Module, SupportsPP): if is_pp_missing_parameter(name, self): continue # Remapping the name of FP8 kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: + remapped_name = maybe_remap_kv_scale_name(name, params_dict) + if remapped_name is None: continue + name = remapped_name param = params_dict[name] diff --git a/vllm/model_executor/models/exaone4_5.py b/vllm/model_executor/models/exaone4_5.py index 676faa9d4f..b325c6a304 100644 --- a/vllm/model_executor/models/exaone4_5.py +++ b/vllm/model_executor/models/exaone4_5.py @@ -332,6 +332,7 @@ class Exaone4_5_ForConditionalGeneration(Qwen2_5_VLForConditionalGeneration): config: Exaone4_5_Config = vllm_config.model_config.hf_config self.vllm_config = vllm_config multimodal_config = vllm_config.model_config.multimodal_config + assert multimodal_config is not None self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data" self.config = config diff --git a/vllm/model_executor/models/exaone4_5_mtp.py b/vllm/model_executor/models/exaone4_5_mtp.py index 80e1903f9d..d9c7960010 100644 --- a/vllm/model_executor/models/exaone4_5_mtp.py +++ b/vllm/model_executor/models/exaone4_5_mtp.py @@ -48,17 +48,11 @@ class Exaone4_5MultiTokenPredictor(ExaoneMoeMultiTokenPredictor): model_config = vllm_config.model_config quant_config = vllm_config.quant_config - lora_config = vllm_config.lora_config config = model_config.hf_config text_config = config.text_config self.config = config - lora_vocab = ( - (lora_config.lora_extra_vocab_size * (lora_config.max_loras or 1)) - if lora_config - else 0 - ) - self.vocab_size = config.vocab_size + lora_vocab + self.vocab_size = config.vocab_size self.org_vocab_size = config.vocab_size self.mtp_start_layer_idx = text_config.num_hidden_layers @@ -67,7 +61,6 @@ class Exaone4_5MultiTokenPredictor(ExaoneMoeMultiTokenPredictor): self.embed_tokens = VocabParallelEmbedding( self.vocab_size, text_config.hidden_size, - org_num_embeddings=config.vocab_size, ) self.fc = ColumnParallelLinear( diff --git a/vllm/model_executor/models/exaone_moe.py b/vllm/model_executor/models/exaone_moe.py index 6fbeb03af8..4c19f1ed0f 100644 --- a/vllm/model_executor/models/exaone_moe.py +++ b/vllm/model_executor/models/exaone_moe.py @@ -41,7 +41,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.layers.vocab_parallel_embedding import ( - DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding, ) @@ -112,6 +111,7 @@ class ExaoneMoe(nn.Module): self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts self.n_local_physical_experts = self.n_physical_experts // self.ep_size + self.shared_experts: ExaoneMoeGatedMLP | None if getattr(config, "num_shared_experts", 0) > 0: intermediate_size = config.moe_intermediate_size * config.num_shared_experts self.shared_experts = ExaoneMoeGatedMLP( @@ -292,7 +292,7 @@ class ExaoneMoeDecoderLayer(nn.Module): config: PretrainedConfig, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, - is_mtp: bool = None, + is_mtp: bool = False, prefix: str = "", ) -> None: super().__init__() @@ -381,26 +381,19 @@ class ExaoneMoeModel(nn.Module): config = vllm_config.model_config.hf_config cache_config = vllm_config.cache_config quant_config = vllm_config.quant_config - lora_config = vllm_config.lora_config self.num_redundant_experts = ( vllm_config.parallel_config.eplb_config.num_redundant_experts ) self.config = config self.quant_config = quant_config - lora_vocab = ( - (lora_config.lora_extra_vocab_size * (lora_config.max_loras or 1)) - if lora_config - else 0 - ) - self.vocab_size = config.vocab_size + lora_vocab + self.vocab_size = config.vocab_size if get_pp_group().is_first_rank or ( config.tie_word_embeddings and get_pp_group().is_last_rank ): self.embed_tokens = VocabParallelEmbedding( self.vocab_size, config.hidden_size, - org_num_embeddings=config.vocab_size, quant_config=quant_config, ) else: @@ -497,10 +490,8 @@ class ExaoneMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP): super().__init__() config = vllm_config.model_config.hf_config.get_text_config() quant_config = vllm_config.quant_config - lora_config = vllm_config.lora_config self.config = config - self.lora_config = lora_config self.quant_config = quant_config self.model = ExaoneMoeModel( @@ -509,17 +500,9 @@ class ExaoneMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP): ) if get_pp_group().is_last_rank: self.unpadded_vocab_size = config.vocab_size - if lora_config: - self.unpadded_vocab_size += lora_config.lora_extra_vocab_size self.lm_head = ParallelLMHead( self.unpadded_vocab_size, config.hidden_size, - org_num_embeddings=config.vocab_size, - padding_size=DEFAULT_VOCAB_PADDING_SIZE - # We need bigger padding if using lora for kernel - # compatibility - if not lora_config - else lora_config.lora_vocab_padding_size, quant_config=quant_config, ) if config.tie_word_embeddings: diff --git a/vllm/model_executor/models/exaone_moe_mtp.py b/vllm/model_executor/models/exaone_moe_mtp.py index ebe1f8cbae..108fa37df8 100644 --- a/vllm/model_executor/models/exaone_moe_mtp.py +++ b/vllm/model_executor/models/exaone_moe_mtp.py @@ -46,16 +46,10 @@ class ExaoneMoeMultiTokenPredictor(nn.Module): model_config = vllm_config.model_config quant_config = vllm_config.quant_config - lora_config = vllm_config.lora_config config = model_config.hf_config self.config = config - lora_vocab = ( - (lora_config.lora_extra_vocab_size * (lora_config.max_loras or 1)) - if lora_config - else 0 - ) - self.vocab_size = config.vocab_size + lora_vocab + self.vocab_size = config.vocab_size self.org_vocab_size = config.vocab_size self.mtp_start_layer_idx = config.num_hidden_layers @@ -64,7 +58,6 @@ class ExaoneMoeMultiTokenPredictor(nn.Module): self.embed_tokens = VocabParallelEmbedding( self.vocab_size, config.hidden_size, - org_num_embeddings=config.vocab_size, ) self.fc = ColumnParallelLinear( diff --git a/vllm/model_executor/models/extract_hidden_states.py b/vllm/model_executor/models/extract_hidden_states.py index d94db2c692..c11c64fcba 100644 --- a/vllm/model_executor/models/extract_hidden_states.py +++ b/vllm/model_executor/models/extract_hidden_states.py @@ -231,7 +231,7 @@ class CacheOnlyAttentionLayer(nn.Module, AttentionLayerBase): head_size: int, cache_config: CacheConfig | None = None, prefix: str = "", - attn_type: str = AttentionType.DECODER, + attn_type: AttentionType = AttentionType.DECODER, ): super().__init__() @@ -329,7 +329,9 @@ class ExtractHiddenStatesModel(nn.Module): super().__init__() self.vllm_config = vllm_config - self.hf_config = vllm_config.speculative_config.draft_model_config.hf_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + self.hf_config = speculative_config.draft_model_config.hf_config self.hidden_size = vllm_config.model_config.get_hidden_size() self.target_num_hidden_layers = ( vllm_config.model_config.get_total_num_hidden_layers() diff --git a/vllm/model_executor/models/falcon.py b/vllm/model_executor/models/falcon.py index 97e133a03c..f2eb776b14 100644 --- a/vllm/model_executor/models/falcon.py +++ b/vllm/model_executor/models/falcon.py @@ -413,6 +413,7 @@ class FalconModel(nn.Module): else: hidden_states = self.embed_input_ids(input_ids) else: + assert intermediate_tensors is not None hidden_states = intermediate_tensors["hidden_states"] for layer in islice(self.h, self.start_layer, self.end_layer): hidden_states = layer(positions, hidden_states) diff --git a/vllm/model_executor/models/fireredasr2.py b/vllm/model_executor/models/fireredasr2.py index f652eb7518..b9d9947459 100644 --- a/vllm/model_executor/models/fireredasr2.py +++ b/vllm/model_executor/models/fireredasr2.py @@ -12,7 +12,7 @@ from transformers import ( ) from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions from vllm.config.speech_to_text import SpeechToTextParams from vllm.inputs import MultiModalDataDict, PromptType from vllm.logger import init_logger @@ -216,6 +216,7 @@ class FireRedASR2DummyInputsBuilder(BaseDummyInputsBuilder[FireRedASR2Processing num_audios = mm_counts.get("audio", 0) audio_overrides = mm_options.get("audio") + assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) ret = { "audio": self._get_dummy_audios( diff --git a/vllm/model_executor/models/fireredlid.py b/vllm/model_executor/models/fireredlid.py index 2225d70f9e..7376556b3f 100644 --- a/vllm/model_executor/models/fireredlid.py +++ b/vllm/model_executor/models/fireredlid.py @@ -18,14 +18,13 @@ from __future__ import annotations from collections.abc import Iterable, Mapping, Sequence from typing import Annotated, Literal -import numpy as np import torch from torch import nn from transformers import BatchFeature from vllm.config import ModelConfig, VllmConfig -from vllm.config.multimodal import BaseDummyOptions -from vllm.config.speech_to_text import SpeechToTextConfig +from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions +from vllm.config.speech_to_text import SpeechToTextConfig, SpeechToTextParams from vllm.distributed import get_tensor_model_parallel_world_size from vllm.inputs import MultiModalDataDict, PromptType from vllm.logger import init_logger @@ -431,6 +430,7 @@ class FireRedLIDDummyInputsBuilder(BaseDummyInputsBuilder[FireRedLIDProcessingIn audio_len = feature_extractor.chunk_length * sampling_rate num_audios = mm_counts.get("audio", 0) audio_overrides = mm_options.get("audio") + assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) return { "audio": self._get_dummy_audios( length=audio_len, @@ -747,13 +747,7 @@ class FireRedLIDForConditionalGeneration( @classmethod def get_generation_prompt( cls, - audio: np.ndarray, - stt_config: SpeechToTextConfig, - model_config: ModelConfig, - language: str | None, - task_type: Literal["transcribe", "translate"], - request_prompt: str, - to_language: str | None, + stt_params: SpeechToTextParams, ) -> PromptType: """Build the prompt for the FireRedLID encoder-decoder model. @@ -764,7 +758,10 @@ class FireRedLIDForConditionalGeneration( "encoder_prompt": { "prompt": "", "multi_modal_data": { - "audio": (audio, int(stt_config.sample_rate)), + "audio": ( + stt_params.audio, + int(stt_params.stt_config.sample_rate), + ), }, }, "decoder_prompt": { diff --git a/vllm/model_executor/models/funasr.py b/vllm/model_executor/models/funasr.py index e15cde2d75..b7981f42d3 100644 --- a/vllm/model_executor/models/funasr.py +++ b/vllm/model_executor/models/funasr.py @@ -14,7 +14,7 @@ from transformers import ( ) from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions from vllm.config.speech_to_text import SpeechToTextParams from vllm.distributed import get_tensor_model_parallel_world_size from vllm.inputs import MultiModalDataDict, PromptType @@ -261,8 +261,8 @@ class SinusoidalPositionEncoder(torch.nn.Module): def encode( self, - positions: torch.Tensor = None, - depth: int = None, + positions: torch.Tensor, + depth: int, dtype: torch.dtype = torch.float32, ): batch_size = positions.size(0) @@ -573,8 +573,8 @@ class Transformer(nn.Module): ] ) - def forward(self, hidden_states: torch.Tensor, ilens: int = 0): - max_len = max(ilens) + def forward(self, hidden_states: torch.Tensor, ilens: torch.Tensor): + max_len = ilens.max() hidden_states = hidden_states[:, :max_len, :] batch_size, seq_len, dim = hidden_states.size() chunk_num = (seq_len - 1) // self.k + 1 @@ -682,6 +682,8 @@ class FunASRModel(nn.Module): self.feat_permute = False if self.feat_permute: + if not isinstance(speech, torch.Tensor): + raise NotImplementedError("List audio inputs cannot be permuted") encoder_out, encoder_out_lens = self.encoder.audio_encoder( speech.permute(0, 2, 1), speech_lengths ) @@ -739,6 +741,7 @@ class FunASRDummyInputsBuilder(BaseDummyInputsBuilder[FunASRProcessingInfo]): num_audios = mm_counts.get("audio", 0) audio_overrides = mm_options.get("audio") + assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) return { "audio": self._get_dummy_audios( diff --git a/vllm/model_executor/models/funaudiochat.py b/vllm/model_executor/models/funaudiochat.py index 8a0cf1364c..4b8ba82d70 100644 --- a/vllm/model_executor/models/funaudiochat.py +++ b/vllm/model_executor/models/funaudiochat.py @@ -26,7 +26,7 @@ from transformers.feature_extraction_utils import BatchFeature from transformers.modeling_outputs import BaseModelOutput from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear @@ -600,6 +600,7 @@ class FunAudioChatDummyInputsBuilder( num_audios = int(mm_counts.get("audio", 0)) audio_overrides = mm_options.get("audio") + assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) return { "audio": self._get_dummy_audios( length=audio_len, @@ -635,6 +636,7 @@ class FunAudioChatMultiModalProcessor( processed_data = BatchFeature({"input_ids": input_ids}) processed_data.update(passthrough_data) return processed_data + assert isinstance(audios, Sequence) feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs) sr = int(feature_extractor.sampling_rate)