From 70021a0081cc10d8862f7458c36353b72ea9b5f4 Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:47:16 -0400 Subject: [PATCH] [Mypy Fix] Mypy fix for "vllm/model_executor/models/[gG]" (#53616) Signed-off-by: yewentao256 --- tools/pre_commit/mypy.py | 1 - vllm/model_executor/models/deepseek_v2.py | 4 +- vllm/model_executor/models/gemma.py | 1 + vllm/model_executor/models/gemma3_mm.py | 2 + vllm/model_executor/models/gemma3n.py | 2 +- vllm/model_executor/models/gemma3n_mm.py | 41 ++++++--- vllm/model_executor/models/gemma4.py | 38 +++++++-- vllm/model_executor/models/gemma4_dspark.py | 6 +- vllm/model_executor/models/gemma4_mm.py | 72 ++++++++++++---- vllm/model_executor/models/gemma4_mtp.py | 13 ++- vllm/model_executor/models/gemma4_unified.py | 28 +++---- vllm/model_executor/models/glm4_1v.py | 58 +++++++++---- vllm/model_executor/models/glm4_moe.py | 2 + vllm/model_executor/models/glm4_moe_lite.py | 20 ++--- .../models/glm4_moe_lite_mtp.py | 19 +++-- vllm/model_executor/models/glm4_moe_mtp.py | 1 + vllm/model_executor/models/glm4v.py | 21 ++++- vllm/model_executor/models/glm_ocr.py | 2 + vllm/model_executor/models/glm_ocr_mtp.py | 11 ++- vllm/model_executor/models/glmasr.py | 19 ++++- vllm/model_executor/models/gpt_j.py | 1 + vllm/model_executor/models/gpt_neox.py | 1 + vllm/model_executor/models/gpt_oss.py | 83 ++++++++++++++----- vllm/model_executor/models/granite4_vision.py | 2 + vllm/model_executor/models/granite_speech.py | 16 +++- .../model_executor/models/granitemoehybrid.py | 9 +- 26 files changed, 350 insertions(+), 123 deletions(-) diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index ab220da8a3..dba3c2afd0 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -112,7 +112,6 @@ SEPARATE_GROUPS = [ ] EXCLUDE = [ - r"vllm/model_executor/models/[gG]", r"vllm/model_executor/models/[hH]", r"vllm/model_executor/models/[iI]", r"vllm/model_executor/models/[jJ]", diff --git a/vllm/model_executor/models/deepseek_v2.py b/vllm/model_executor/models/deepseek_v2.py index 36f9d0bb7b..c45c907ada 100644 --- a/vllm/model_executor/models/deepseek_v2.py +++ b/vllm/model_executor/models/deepseek_v2.py @@ -457,7 +457,7 @@ class DeepseekV2Attention(nn.Module): qk_nope_head_dim: int, qk_rope_head_dim: int, v_head_dim: int, - q_lora_rank: int, + q_lora_rank: int | None, kv_lora_rank: int, max_position_embeddings: int = 8192, cache_config: CacheConfig | None = None, @@ -495,7 +495,7 @@ class DeepseekV2Attention(nn.Module): ) self.q_a_layernorm = RMSNorm(self.q_lora_rank, eps=config.rms_norm_eps) self.q_b_proj = ColumnParallelLinear( - q_lora_rank, + self.q_lora_rank, self.num_heads * self.qk_head_dim, bias=False, quant_config=quant_config, diff --git a/vllm/model_executor/models/gemma.py b/vllm/model_executor/models/gemma.py index 7058290865..983822c158 100644 --- a/vllm/model_executor/models/gemma.py +++ b/vllm/model_executor/models/gemma.py @@ -311,6 +311,7 @@ class GemmaModel(nn.Module): hidden_states = self.embed_input_ids(input_ids) residual = None else: + assert intermediate_tensors is not None hidden_states = intermediate_tensors["hidden_states"] residual = intermediate_tensors["residual"] for layer in islice(self.layers, self.start_layer, self.end_layer): diff --git a/vllm/model_executor/models/gemma3_mm.py b/vllm/model_executor/models/gemma3_mm.py index 0808422801..114085921e 100644 --- a/vllm/model_executor/models/gemma3_mm.py +++ b/vllm/model_executor/models/gemma3_mm.py @@ -852,7 +852,9 @@ class Gemma3ForConditionalGeneration( def encoder_eager_forward( self, mm_kwargs: dict[str, Any], + path: str = "default", ) -> torch.Tensor: image_input = self._parse_and_validate_image_input(**mm_kwargs) + assert image_input is not None results = self._process_image_input(image_input) return torch.cat(results, dim=0) diff --git a/vllm/model_executor/models/gemma3n.py b/vllm/model_executor/models/gemma3n.py index 6e73ad8baa..06872c1ef1 100644 --- a/vllm/model_executor/models/gemma3n.py +++ b/vllm/model_executor/models/gemma3n.py @@ -77,7 +77,7 @@ class Gemma3nAltUp(nn.Module): altup_num_inputs: int, altup_coef_clip: float, altup_active_idx: int, - quant_config: QuantizationConfig, + quant_config: QuantizationConfig | None, prefix: str, ): super().__init__() diff --git a/vllm/model_executor/models/gemma3n_mm.py b/vllm/model_executor/models/gemma3n_mm.py index 4cbaed8d9b..1c05c165dc 100644 --- a/vllm/model_executor/models/gemma3n_mm.py +++ b/vllm/model_executor/models/gemma3n_mm.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import inspect -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence, Sized from typing import Annotated, Any, Literal import torch @@ -18,7 +18,7 @@ from transformers.models.gemma3n import ( from transformers.models.siglip import SiglipImageProcessorFast from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions from vllm.config.speech_to_text import SpeechToTextParams from vllm.inputs import MultiModalDataDict, PromptType, TextPrompt from vllm.logger import init_logger @@ -115,8 +115,10 @@ def batch_audio_features( The `(bn, s_max, f)` features and their `(bn, s_max)` mask. """ if isinstance(input_features, torch.Tensor): + assert isinstance(input_features_mask, torch.Tensor) return input_features.squeeze(1), input_features_mask.squeeze(1) + assert isinstance(input_features_mask, list) max_len = max(features.shape[0] for features in input_features) batched_features = input_features[0].new_zeros( (len(input_features), max_len, input_features[0].shape[-1]) @@ -248,6 +250,7 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]): image_overrides = mm_options.get("image") audio_overrides = mm_options.get("audio") + assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) return { "image": self._get_dummy_images( @@ -293,10 +296,14 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]) # output (and thus its cache entry) is independent of the other # items in the batch. `batch_audio_features` re-pads the # per-item features so audio_tower can still run batched. - num_frames = [ - self._get_num_audio_frames_alone(len(audio)) - for audio in mm_data["audio"] - ] + audios = mm_data["audio"] + if not isinstance(audios, Sequence): + raise TypeError("audio data must be a sequence") + num_frames = [] + for audio in audios: + if not isinstance(audio, Sized): + raise TypeError("each audio item must have a length") + num_frames.append(self._get_num_audio_frames_alone(len(audio))) processed_data["input_features_padded"] = [ features[:n] for features, n in zip(processed_data.pop("input_features"), num_frames) @@ -685,7 +692,9 @@ class Gemma3nForConditionalGeneration( ) def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: - mm_input_by_modality = {} + mm_input_by_modality: dict[ + str, Gemma3nImageInputs | Gemma3nAudioInputs | None + ] = {} # Preserve the order of modalities if there are multiple of them # from the order of kwargs. @@ -852,7 +861,13 @@ class Gemma3nForConditionalGeneration( # select a chunk of pre-allocated PLEs. During normal execution, # `embed_input_ids` is called before forward, hence this slice # will contain PLEs computed from the actual input_ids. - per_layer_inputs = self.per_layer_embeddings[: inputs_embeds.shape[0]] + if inputs_embeds is not None: + num_tokens = inputs_embeds.shape[0] + elif intermediate_tensors is not None: + num_tokens = intermediate_tensors["hidden_states"].shape[0] + else: + raise ValueError("inputs_embeds is required on the first PP rank") + per_layer_inputs = self.per_layer_embeddings[:num_tokens] hidden_states = self.language_model.model( input_ids, @@ -913,9 +928,15 @@ class Gemma3nForConditionalGeneration( prompt += " this audio" # We assume the language is a valid ISO 639-1 code. - full_lang_name = cls.supported_languages.get(language, "") + full_lang_name = ( + cls.supported_languages.get(language, "") if language is not None else "" + ) # Translation only for now - full_lang_name_to = cls.supported_languages.get(to_language, "") + full_lang_name_to = ( + cls.supported_languages.get(to_language, "") + if to_language is not None + else "" + ) if task_type == "transcribe" and full_lang_name: prompt += f" into {full_lang_name}" diff --git a/vllm/model_executor/models/gemma4.py b/vllm/model_executor/models/gemma4.py index 1e3b25d88e..7f850f4960 100644 --- a/vllm/model_executor/models/gemma4.py +++ b/vllm/model_executor/models/gemma4.py @@ -352,15 +352,19 @@ class Gemma4MoE(nn.Module): ) # MoERunner experts with custom Gemma4 routing + intermediate_size = getattr( + config, + "moe_intermediate_size", + getattr(config, "expert_intermediate_size", None), + ) + if intermediate_size is None: + raise ValueError("Gemma4 MoE requires an expert intermediate size") + self.experts = FusedMoEFactory( num_experts=config.num_experts, top_k=config.top_k_experts, hidden_size=config.hidden_size, - intermediate_size=getattr( - config, - "moe_intermediate_size", - getattr(config, "expert_intermediate_size", None), - ), + intermediate_size=intermediate_size, renormalize=True, quant_config=quant_config, prefix=f"{prefix}.experts", @@ -624,6 +628,11 @@ class Gemma4DecoderLayer(nn.Module): self.enable_moe_block = getattr(config, "enable_moe_block", False) or getattr( config, "use_second_mlp_block", False ) + self.router: Gemma4Router | None + self.moe: Gemma4MoE | None + self.post_feedforward_layernorm_1: RMSNorm | None + self.post_feedforward_layernorm_2: RMSNorm | None + self.pre_feedforward_layernorm_2: RMSNorm | None if self.enable_moe_block: self.router = Gemma4Router( config, @@ -652,6 +661,9 @@ class Gemma4DecoderLayer(nn.Module): self.pre_feedforward_layernorm_2 = None # Per-Layer Embedding (PLE) components — present in each decoder layer + self.per_layer_input_gate: ReplicatedLinear | None + self.per_layer_projection: ReplicatedLinear | None + self.post_per_layer_input_norm: RMSNorm | None if ( self.hidden_size_per_layer_input is not None and self.hidden_size_per_layer_input > 0 @@ -716,6 +728,11 @@ class Gemma4DecoderLayer(nn.Module): hidden_states = self.mlp(hidden_states) if self.enable_moe_block: + assert self.post_feedforward_layernorm_1 is not None + assert self.pre_feedforward_layernorm_2 is not None + assert self.router is not None + assert self.moe is not None + assert self.post_feedforward_layernorm_2 is not None hidden_states_1 = self.post_feedforward_layernorm_1(hidden_states) hidden_states_2 = self.pre_feedforward_layernorm_2(residual) @@ -731,6 +748,8 @@ class Gemma4DecoderLayer(nn.Module): # Apply PLE (Per-Layer Embedding) if configured if per_layer_input is not None and self.per_layer_input_gate is not None: + assert self.per_layer_projection is not None + assert self.post_per_layer_input_norm is not None gate = self.per_layer_input_gate(hidden_states) gate = torch.nn.functional.gelu(gate, approximate="tanh") gated_per_layer = gate * per_layer_input @@ -865,6 +884,7 @@ class Gemma4SelfDecoderLayers(nn.Module): """ if self.per_layer_model_projection is None: return None + assert self.per_layer_projection_norm is not None per_layer_projection = self.per_layer_model_projection(inputs_embeds) per_layer_projection = per_layer_projection * self.per_layer_projection_scale per_layer_projection = per_layer_projection.reshape( @@ -971,6 +991,9 @@ class Gemma4Model(nn.Module, EagleModelMixin): ) # Per-Layer Embedding (PLE) components + self.embed_tokens_per_layer: VocabParallelEmbedding | None + self.per_layer_model_projection: ColumnParallelLinear | None + self.per_layer_projection_norm: RMSNorm | None if ( self.hidden_size_per_layer_input is not None and self.hidden_size_per_layer_input > 0 @@ -1472,9 +1495,10 @@ class Gemma4Model(nn.Module, EagleModelMixin): else: if name.endswith(".bias") and name not in params_dict: continue - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: + remapped_name = maybe_remap_kv_scale_name(name, params_dict) + if remapped_name is None: continue + name = remapped_name if is_pp_missing_parameter(name, self): continue # Skip if name doesn't exist in params_dict (e.g., individual diff --git a/vllm/model_executor/models/gemma4_dspark.py b/vllm/model_executor/models/gemma4_dspark.py index f19c3857d1..6b6d5d2731 100644 --- a/vllm/model_executor/models/gemma4_dspark.py +++ b/vllm/model_executor/models/gemma4_dspark.py @@ -89,7 +89,11 @@ class Gemma4DSparkAttention(Gemma4MTPAttention): ) -> tuple[torch.Tensor, torch.Tensor]: k, _ = self.k_proj(hidden_states) k_normed = self.k_norm(k.unflatten(-1, (self.num_kv_heads, self.head_dim))) - v_src = k if self.use_k_eq_v else self.v_proj(hidden_states)[0] + if self.use_k_eq_v: + v_src = k + else: + assert self.v_proj is not None + v_src = self.v_proj(hidden_states)[0] v_normed = self.v_norm(v_src.unflatten(-1, (self.num_kv_heads, self.head_dim))) return k_normed.flatten(-2, -1), v_normed.flatten(-2, -1) diff --git a/vllm/model_executor/models/gemma4_mm.py b/vllm/model_executor/models/gemma4_mm.py index c636897aae..428ab9ec16 100644 --- a/vllm/model_executor/models/gemma4_mm.py +++ b/vllm/model_executor/models/gemma4_mm.py @@ -35,7 +35,11 @@ from transformers.models.gemma4.configuration_gemma4 import ( from vllm.config import VllmConfig from vllm.config.model import get_served_model_name -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions +from vllm.config.multimodal import ( + AudioDummyOptions, + BaseDummyOptions, + VideoDummyOptions, +) from vllm.inputs import MultiModalDataDict from vllm.logger import init_logger from vllm.model_executor.layers.layernorm import RMSNorm @@ -189,6 +193,7 @@ class Gemma4VideoInputs(TensorSchema): torch.Tensor, TensorShape("bn", "np", 2), ] + video_frame_counts: Annotated[torch.Tensor, TensorShape("nv")] # --------------------------------------------------------------------------- @@ -490,8 +495,10 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]): image_overrides = mm_options.get("image") if mm_options else None audio_overrides = mm_options.get("audio") if mm_options else None video_overrides = mm_options.get("video") if mm_options else None + assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) + assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions) - data: MultiModalDataDict = { + data: dict[str, Any] = { "image": self._get_dummy_images( width=img_width, height=img_height, @@ -592,6 +599,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]): # same vision tower, matching transformers processing_gemma4.py. video_outputs: dict[str, Any] = {} if videos := mm_data.pop("videos", []): + assert isinstance(videos, list) processor = self.info.get_hf_processor() all_video_pixel_values: list[torch.Tensor] = [] @@ -858,8 +866,29 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]): def get_replacement_video(item_idx: int): out_item = out_mm_kwargs["video"][item_idx] - timestamps = out_item["video_timestamps"].data - num_soft = out_item["video_num_soft_tokens"].data + timestamps_data = out_item["video_timestamps"].data + if isinstance(timestamps_data, torch.Tensor): + timestamps_data = timestamps_data.tolist() + if not isinstance(timestamps_data, list): + raise TypeError("video timestamps must be a list") + timestamps: list[float] = [] + for timestamp in timestamps_data: + if not isinstance(timestamp, (int, float)): + raise TypeError("each video timestamp must be numeric") + timestamps.append(float(timestamp)) + + num_soft_data = out_item["video_num_soft_tokens"].data + if isinstance(num_soft_data, torch.Tensor): + num_soft_data = num_soft_data.tolist() + if not isinstance(num_soft_data, list): + raise TypeError("video soft-token counts must be a list") + num_soft: list[int] = [] + for count in num_soft_data: + if not isinstance(count, int): + raise TypeError( + "each video soft-token count must be an integer" + ) + num_soft.append(count) return self.info.get_video_repl( timestamps=timestamps, num_soft_tokens_per_frame=num_soft, @@ -1038,6 +1067,7 @@ class Gemma4ForConditionalGeneration( # (Marlin, FP8, …) require dimensions divisible by 64, which # the vision tower (intermediate_size=4304) does not satisfy. # TODO(mgoin): remove this by fixing kernel padding. + tower_quant: QuantizationConfig | None if quant_config and quant_config.get_name() in [ "bitsandbytes", "torchao", @@ -1068,6 +1098,7 @@ class Gemma4ForConditionalGeneration( ) # ---- Audio tower (variants with audio_config) ---- + self.embed_audio: Gemma4MultimodalEmbedder | None if config.audio_config is not None: with self._mark_tower_model(vllm_config, "audio"): self.audio_tower = AutoModel.from_config(config=config.audio_config) @@ -1177,22 +1208,24 @@ class Gemma4ForConditionalGeneration( def _parse_and_validate_video_input( self, **kwargs: object - ) -> dict[str, torch.Tensor] | None: + ) -> Gemma4VideoInputs | None: pixel_values_videos = kwargs.pop("pixel_values_videos", None) pixel_position_ids_videos = kwargs.pop("pixel_position_ids_videos", None) video_frame_counts = kwargs.pop("video_frame_counts", None) if pixel_values_videos is None: return None - return { - "pixel_values_videos": pixel_values_videos, - "pixel_position_ids_videos": pixel_position_ids_videos, - "video_frame_counts": video_frame_counts, - } + return Gemma4VideoInputs( + pixel_values_videos=pixel_values_videos, + pixel_position_ids_videos=pixel_position_ids_videos, + video_frame_counts=video_frame_counts, + ) def _parse_and_validate_multimodal_inputs( self, **kwargs: object ) -> dict[str, Gemma4ImageInputs | Gemma4AudioInputs | Gemma4VideoInputs | None]: - mm_input_by_modality = {} + mm_input_by_modality: dict[ + str, Gemma4ImageInputs | Gemma4AudioInputs | Gemma4VideoInputs | None + ] = {} for input_key in list(kwargs): if ( input_key in ("pixel_values", "image_embeds") @@ -1419,7 +1452,7 @@ class Gemma4ForConditionalGeneration( def _process_video_input( self, - video_input: dict[str, torch.Tensor], + video_input: Gemma4VideoInputs, ) -> list[torch.Tensor]: """Batch-encode video frames through the vision tower. @@ -1543,6 +1576,8 @@ class Gemma4ForConditionalGeneration( audio_input["input_features_mask"], ) + assert self.audio_tower is not None + assert self.embed_audio is not None # Run audio tower — mask convention: True=valid, False=padding. audio_outputs = self.audio_tower(input_features, input_features_mask) if isinstance(audio_outputs, tuple): @@ -1576,14 +1611,17 @@ class Gemma4ForConditionalGeneration( if multimodal_input is None: continue if modality == "image": + assert isinstance(multimodal_input, Gemma4ImageInputs) multimodal_embeddings.extend( self._process_image_input(multimodal_input) ) elif modality == "video": + assert isinstance(multimodal_input, Gemma4VideoInputs) multimodal_embeddings.extend( self._process_video_input(multimodal_input) ) elif modality == "audio": + assert isinstance(multimodal_input, Gemma4AudioInputs) multimodal_embeddings.extend( self._process_audio_input(multimodal_input) ) @@ -2204,13 +2242,15 @@ class Gemma4ForConditionalGeneration( mm_kwargs: MultiModalKwargsItem | None, num_mm_embeds: int, ) -> tuple[int, int | None]: + tower_tokens: int | None = None + connector_tokens: int | None = None if modality in ("image", "video"): vision_config = self.config.vision_config pooling_k2 = vision_config.pooling_kernel_size**2 if modality == "image": pixel_values_key = "pixel_values" - max_soft_tokens = vision_config.default_output_length + max_soft_tokens = int(vision_config.default_output_length) mm_processor_kwargs = getattr( getattr(self, "multimodal_config", None), "mm_processor_kwargs", @@ -2241,7 +2281,7 @@ class Gemma4ForConditionalGeneration( * pooling_k2 ) - if modality == "audio": + elif modality == "audio": tower_tokens = num_mm_embeds connector_tokens = num_mm_embeds @@ -2255,6 +2295,10 @@ class Gemma4ForConditionalGeneration( tower_tokens = audio_tokens connector_tokens = audio_tokens + else: + raise ValueError(f"Unsupported modality: {modality}") + + assert tower_tokens is not None return tower_tokens, connector_tokens @classmethod diff --git a/vllm/model_executor/models/gemma4_mtp.py b/vllm/model_executor/models/gemma4_mtp.py index 9fd2b1f164..8e135d84c0 100644 --- a/vllm/model_executor/models/gemma4_mtp.py +++ b/vllm/model_executor/models/gemma4_mtp.py @@ -345,7 +345,9 @@ class Gemma4MultiTokenPredictor(nn.Module): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() - config = vllm_config.speculative_config.draft_model_config.hf_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + config = speculative_config.draft_model_config.hf_config text_config = _get_text_config(config) quant_config = get_draft_quant_config(vllm_config) self.config = text_config @@ -467,7 +469,9 @@ class Gemma4MTP(nn.Module): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() - config = vllm_config.speculative_config.draft_model_config.hf_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + config = speculative_config.draft_model_config.hf_config text_config = _get_text_config(config) self.quant_config = get_draft_quant_config(vllm_config) self.config = config @@ -496,6 +500,7 @@ class Gemma4MTP(nn.Module): soft_cap=getattr(text_config, "final_logit_softcapping", None), ) + self.masked_embedding: Gemma4MTPMaskedEmbedder | None if getattr(config, "use_ordered_embeddings", False): num_centroids = getattr(config, "num_centroids", 2048) top_k = getattr(config, "centroid_intermediate_top_k", 32) @@ -516,7 +521,7 @@ class Gemma4MTP(nn.Module): else: self.masked_embedding = None - draft_cfg = vllm_config.speculative_config.draft_model_config + draft_cfg = speculative_config.draft_model_config gen_cfg = draft_cfg.try_get_generation_config() self._suppress_token_ids = gen_cfg.get("suppress_tokens") if gen_cfg else None @@ -545,6 +550,7 @@ class Gemma4MTP(nn.Module): def _get_full_lm_head_weight(self) -> torch.Tensor: if self._stable_full_lm_head_weight is not None: return self._stable_full_lm_head_weight + assert self.masked_embedding is not None lm_head_weight = self.lm_head.weight tp_size = get_tensor_model_parallel_world_size() if tp_size > 1: @@ -579,6 +585,7 @@ class Gemma4MTP(nn.Module): hidden_states: torch.Tensor, ) -> torch.Tensor: """Sparse argmax via centroids masking. Returns token IDs directly.""" + assert self.masked_embedding is not None return self.masked_embedding.get_top_tokens( hidden_states, self._get_full_lm_head_weight(), diff --git a/vllm/model_executor/models/gemma4_unified.py b/vllm/model_executor/models/gemma4_unified.py index f41a36adfd..7c9aaa8499 100644 --- a/vllm/model_executor/models/gemma4_unified.py +++ b/vllm/model_executor/models/gemma4_unified.py @@ -44,6 +44,7 @@ from vllm.model_executor.models.gemma4_mm import ( Gemma4MultimodalEmbedder, Gemma4MultiModalProcessor, Gemma4ProcessingInfo, + Gemma4VideoInputs, _get_max_soft_tokens, ) from vllm.model_executor.models.module_mapping import MultiModelKeys @@ -265,22 +266,16 @@ class Gemma4UnifiedForConditionalGeneration(Gemma4ForConditionalGeneration): self.audio_tower = None # ---- Encoder-free vision embedder ---- - self.vision_embedder = ( - Gemma4UnifiedVisionEmbedder( - config.vision_config, - quant_config=quant_config, - prefix=maybe_prefix(prefix, "vision_embedder"), - ) - if config.vision_config is not None - else None + if config.vision_config is None: + raise ValueError("Gemma4 Unified requires a vision configuration") + self.vision_embedder = Gemma4UnifiedVisionEmbedder( + config.vision_config, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "vision_embedder"), ) - self.embed_vision = ( - Gemma4MultimodalEmbedder( - config.vision_config, - config.text_config, - ) - if config.vision_config is not None - else None + self.embed_vision = Gemma4MultimodalEmbedder( + config.vision_config, + config.text_config, ) # ---- Encoder-free audio embedder ---- @@ -382,7 +377,7 @@ class Gemma4UnifiedForConditionalGeneration(Gemma4ForConditionalGeneration): def _process_video_input( self, - video_input: dict[str, torch.Tensor], + video_input: Gemma4VideoInputs, ) -> list[torch.Tensor]: """Project video frames to LM space, one frame at a time. @@ -429,6 +424,7 @@ class Gemma4UnifiedForConditionalGeneration(Gemma4ForConditionalGeneration): audio_input["input_features_mask"], ) + assert self.embed_audio is not None target_dtype = self.embed_audio.embedding_projection.weight.dtype audio_features = self.embed_audio(input_features.to(target_dtype)) per_audio: list[torch.Tensor] = [] diff --git a/vllm/model_executor/models/glm4_1v.py b/vllm/model_executor/models/glm4_1v.py index 8dd0beb66c..96bed8b6ee 100644 --- a/vllm/model_executor/models/glm4_1v.py +++ b/vllm/model_executor/models/glm4_1v.py @@ -1393,9 +1393,15 @@ class Glm4vProcessingInfo(BaseProcessingInfo): return list(timestamps_list) def _get_video_frame_embed_token_id(self, hf_processor: object) -> int: - if isinstance(hf_processor, Glm4vProcessor) or TRANSFORMERS_WITH_GA: - return hf_processor.image_token_id - return hf_processor.video_token_id + attr = ( + "image_token_id" + if isinstance(hf_processor, Glm4vProcessor) or TRANSFORMERS_WITH_GA + else "video_token_id" + ) + token_id = getattr(hf_processor, attr, None) + if not isinstance(token_id, int): + raise ValueError(f"Processor has no valid {attr}") + return token_id def _construct_video_placeholder( self, @@ -1479,6 +1485,7 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]): image_overrides = mm_options.get("image") video_overrides = mm_options.get("video") + assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions) return { "image": self._get_dummy_images( @@ -1519,7 +1526,7 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]): ) videos = [v.copy() for v in videos] - video_items = [] + video_items: list[VideoItem] = [] for video in videos: video_num_frames = video.shape[0] video_metadata = { @@ -1611,16 +1618,14 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): processed_data.update(passthrough_data) return processed_data - if ( - "videos" in mm_data - and isinstance(mm_data["videos"], list) - and len(mm_data["videos"]) > 0 - ): + videos = mm_data.get("videos") + if isinstance(videos, list) and videos: video_grid_thw_lst = [] pixel_values_videos_lst = [] frame_embed_token_id = self.info._get_video_frame_embed_token_id(processor) swap_video_frame_tokens = frame_embed_token_id == processor.image_token_id - for item in mm_data.pop("videos", []): + mm_data.pop("videos") + for item in videos: video_array, metadata = item # don't update mm_kwargs inplace @@ -1794,6 +1799,7 @@ class Glm4vForConditionalGeneration( self.config = config self.model_config = vllm_config.model_config self.multimodal_config = multimodal_config + assert multimodal_config is not None self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data" self.is_multimodal_pruning_enabled = ( multimodal_config.is_multimodal_pruning_enabled() @@ -1850,6 +1856,7 @@ class Glm4vForConditionalGeneration( image_embeds=image_embeds, image_grid_thw=image_grid_thw, ) + raise AssertionError("image input must contain pixels or embeddings") def _parse_and_validate_video_input( self, **kwargs: object @@ -1874,6 +1881,7 @@ class Glm4vForConditionalGeneration( video_embeds=video_embeds, video_grid_thw=video_grid_thw, ) + raise AssertionError("video input must contain pixels or embeddings") def _process_image_input( self, image_input: Glm4vImageInputs @@ -1996,12 +2004,12 @@ class Glm4vForConditionalGeneration( def _get_grid_thw_by_modality( self, mm_kwargs: dict[str, Any], - ) -> list[tuple[int, int, int]]: + ) -> list[list[int]]: grid_thw_key = f"{self.get_input_modality(mm_kwargs)}_grid_thw" grid_thw = mm_kwargs[grid_thw_key] if not isinstance(grid_thw, list): grid_thw = grid_thw.tolist() - return grid_thw + return [list(row) for row in grid_thw] def get_encoder_cudagraph_item_specs( self, @@ -2188,7 +2196,7 @@ class Glm4vForConditionalGeneration( return self.visual(pixel_values, grid_thw) def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: - mm_input_by_modality = {} + mm_input_by_modality: dict[str, Glm4vImageInputs | Glm4vVideoInputs | None] = {} # Preserve the order of modalities if there are multiple of them # from the order of kwargs. @@ -2223,9 +2231,17 @@ class Glm4vForConditionalGeneration( for modality in mm_input_by_modality: multimodal_input = mm_input_by_modality[modality] if modality == "image": + assert isinstance( + multimodal_input, + (Glm4vImagePixelInputs, Glm4vImageEmbeddingInputs), + ) image_embeddings = self._process_image_input(multimodal_input) multimodal_embeddings += tuple(image_embeddings) if modality == "video": + assert isinstance( + multimodal_input, + (Glm4vVideoPixelInputs, Glm4vVideoEmbeddingInputs), + ) video_embeddings = self._process_video_input(multimodal_input) multimodal_embeddings += tuple(video_embeddings) return multimodal_embeddings @@ -2238,14 +2254,26 @@ class Glm4vForConditionalGeneration( for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset): embed_ranges = mm_feature.mm_position.extract_embeds_range() if mm_feature.modality == "image": - t, h, w = mm_feature.data["image_grid_thw"].data.tolist() + feature_data = mm_feature.data + assert feature_data is not None + grid_item = feature_data.get("image_grid_thw") + assert grid_item is not None + grid_data = grid_item.data + assert isinstance(grid_data, torch.Tensor) + t, h, w = grid_data.tolist() assert t == 1, f"Image must have 1 frame, got {t}" assert len(embed_ranges) == 1 offset, end = embed_ranges[0] assert end - offset + 1 == h * w // spatial_merge_size**2 yield offset, t, h // spatial_merge_size, w // spatial_merge_size elif mm_feature.modality == "video": - t, h, w = mm_feature.data["video_grid_thw"].data.tolist() + feature_data = mm_feature.data + assert feature_data is not None + grid_item = feature_data.get("video_grid_thw") + assert grid_item is not None + grid_data = grid_item.data + assert isinstance(grid_data, torch.Tensor) + t, h, w = grid_data.tolist() llm_grid_h = h // spatial_merge_size llm_grid_w = w // spatial_merge_size num_tokens_per_frame = llm_grid_h * llm_grid_w diff --git a/vllm/model_executor/models/glm4_moe.py b/vllm/model_executor/models/glm4_moe.py index 2192f74b51..3c72a148d5 100644 --- a/vllm/model_executor/models/glm4_moe.py +++ b/vllm/model_executor/models/glm4_moe.py @@ -502,6 +502,8 @@ class Glm4MoeModel(nn.Module): class Glm4MixtureOfExperts(MixtureOfExperts): + moe_mlp_layers: list[Glm4MoE] + def extract_moe_parameters(self, example_moe: Glm4MoE | None) -> None: if example_moe is None: raise RuntimeError("No Glm4MoE layer found in model.layers.") diff --git a/vllm/model_executor/models/glm4_moe_lite.py b/vllm/model_executor/models/glm4_moe_lite.py index e51b0e231c..d0119719af 100644 --- a/vllm/model_executor/models/glm4_moe_lite.py +++ b/vllm/model_executor/models/glm4_moe_lite.py @@ -74,7 +74,6 @@ from .utils import ( PPMissingLayer, get_spec_layer_idx_from_weight_name, is_pp_missing_parameter, - make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, ) @@ -132,6 +131,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module): v_head_dim = getattr(config, "v_head_dim", 0) kv_lora_rank = getattr(config, "kv_lora_rank", 0) + attn_cls: Callable[..., Glm4MoeLiteMLAAttention | Glm4MoeLiteAttention] if model_config.use_mla: attn_cls = Glm4MoeLiteMLAAttention else: @@ -145,7 +145,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module): qk_nope_head_dim=qk_nope_head_dim, qk_rope_head_dim=qk_rope_head_dim, v_head_dim=v_head_dim, - q_lora_rank=config.q_lora_rank if hasattr(config, "q_lora_rank") else None, + q_lora_rank=getattr(config, "q_lora_rank", None), kv_lora_rank=kv_lora_rank, max_position_embeddings=max_position_embeddings, cache_config=cache_config, @@ -266,9 +266,6 @@ class Glm4MoeLiteModel(nn.Module): self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) else: self.norm = PPMissingLayer() - self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( - ["hidden_states", "residual"], config.hidden_size - ) def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.embed_tokens(input_ids) @@ -454,7 +451,9 @@ class Glm4MoeLiteModel(nn.Module): # param and delegate to its expert-aware weight_loader # with expert_id. for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping + param_name, weight_name, mapping_expert_id, mapping_shard_id = ( + mapping + ) if weight_name not in chunk_name: continue @@ -480,8 +479,8 @@ class Glm4MoeLiteModel(nn.Module): param, weight_to_load, name_mapped, - shard_id=shard_id, - expert_id=expert_id, + shard_id=mapping_shard_id, + expert_id=mapping_expert_id, return_success=True, ) if success: @@ -502,9 +501,10 @@ class Glm4MoeLiteModel(nn.Module): continue # Remapping the name of FP8 kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: + remapped_name = maybe_remap_kv_scale_name(name, params_dict) + if remapped_name is None: continue + name = remapped_name if is_pp_missing_parameter(name, self): continue diff --git a/vllm/model_executor/models/glm4_moe_lite_mtp.py b/vllm/model_executor/models/glm4_moe_lite_mtp.py index 6b28c38b58..1d30a0eaf6 100644 --- a/vllm/model_executor/models/glm4_moe_lite_mtp.py +++ b/vllm/model_executor/models/glm4_moe_lite_mtp.py @@ -85,7 +85,9 @@ class Glm4MoeLiteMultiTokenPredictorLayer(nn.Module): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() - config = vllm_config.speculative_config.draft_model_config.hf_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + config = speculative_config.draft_model_config.hf_config self.config = config quant_config = vllm_config.quant_config @@ -235,7 +237,7 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts): def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.model.embed_input_ids(input_ids) - def forward( + def forward( # type: ignore[override] self, input_ids: torch.Tensor | None, positions: torch.Tensor, @@ -370,7 +372,9 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts): # with expert_id. is_expert_weight = False for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping + param_name, weight_name, mapping_expert_id, mapping_shard_id = ( + mapping + ) if weight_name not in chunk_name: continue @@ -393,8 +397,8 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts): param, weight_to_load, name_mapped, - shard_id=shard_id, - expert_id=expert_id, + shard_id=mapping_shard_id, + expert_id=mapping_expert_id, return_success=True, ) if success: @@ -414,9 +418,10 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts): if name.endswith(".bias") and name not in params_dict: continue - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: + remapped_name = maybe_remap_kv_scale_name(name, params_dict) + if remapped_name is None: continue + name = remapped_name # According to DeepSeek-V3 Technical Report, MTP modules # shares embedding layer. We only load the first weights. diff --git a/vllm/model_executor/models/glm4_moe_mtp.py b/vllm/model_executor/models/glm4_moe_mtp.py index 8af40cf60b..bad4ec8cf3 100644 --- a/vllm/model_executor/models/glm4_moe_mtp.py +++ b/vllm/model_executor/models/glm4_moe_mtp.py @@ -93,6 +93,7 @@ class Glm4MoeMultiTokenPredictorLayer(nn.Module): self.shared_head = SharedHead( config=config, prefix=prefix, quant_config=quant_config ) + assert parallel_config is not None self.enable_eplb = parallel_config.enable_eplb self.mtp_block = Glm4MoeDecoderLayer( config=config, diff --git a/vllm/model_executor/models/glm4v.py b/vllm/model_executor/models/glm4v.py index a8512f49fa..b00e687361 100644 --- a/vllm/model_executor/models/glm4v.py +++ b/vllm/model_executor/models/glm4v.py @@ -582,7 +582,13 @@ class GLM4VForCausalLM( for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset): offset = mm_feature.mm_position.offset if mm_feature.modality == "image": - t, h, w = mm_feature.data["image_grid_thw"].data.tolist() + feature_data = mm_feature.data + assert feature_data is not None + grid_item = feature_data.get("image_grid_thw") + assert grid_item is not None + grid_data = grid_item.data + assert isinstance(grid_data, torch.Tensor) + t, h, w = grid_data.tolist() assert t == 1, f"Image must have 1 frame, got {t}" yield offset, t, h // spatial_merge_size, w // spatial_merge_size else: @@ -625,7 +631,18 @@ class GLM4VForCausalLM( mrope_position_delta = (llm_positions.max() + 1 - len(input_tokens)).item() return torch.from_numpy(llm_positions), mrope_position_delta - embed_input_ids = SupportsMultiModal.embed_input_ids + def embed_input_ids( + self, + input_ids: torch.Tensor, + multimodal_embeddings: MultiModalEmbeddings | None = None, + is_multimodal: torch.Tensor | None = None, + ) -> torch.Tensor: + return SupportsMultiModal.embed_input_ids( + self, + input_ids, + multimodal_embeddings, + is_multimodal=is_multimodal, + ) def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings: image_input = self._parse_and_validate_image_input(**kwargs) diff --git a/vllm/model_executor/models/glm_ocr.py b/vllm/model_executor/models/glm_ocr.py index 9ead8854de..90aca8485d 100644 --- a/vllm/model_executor/models/glm_ocr.py +++ b/vllm/model_executor/models/glm_ocr.py @@ -326,7 +326,9 @@ class GlmOcrVisionTransformer(Glm4vVisionTransformer): self, x: torch.Tensor, grid_thw: torch.Tensor | list[list[int]], + encoder_metadata: dict[str, torch.Tensor] | None = None, ) -> torch.Tensor: + del encoder_metadata if isinstance(grid_thw, list): grid_thw = torch.tensor(grid_thw, dtype=torch.int32) diff --git a/vllm/model_executor/models/glm_ocr_mtp.py b/vllm/model_executor/models/glm_ocr_mtp.py index 0904b44eb2..6c8e208c4b 100644 --- a/vllm/model_executor/models/glm_ocr_mtp.py +++ b/vllm/model_executor/models/glm_ocr_mtp.py @@ -58,7 +58,9 @@ class GlmOcrMultiTokenPredictorLayer(nn.Module): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): nn.Module.__init__(self) - config = vllm_config.speculative_config.draft_model_config.hf_config.text_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + config = speculative_config.draft_model_config.hf_config.text_config self.config = config quant_config = vllm_config.quant_config @@ -144,7 +146,7 @@ class GlmOcrMTP(nn.Module, SupportsPP): def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.model.embed_input_ids(input_ids) - def forward( + def forward( # type: ignore[override] self, input_ids: torch.Tensor, positions: torch.Tensor, @@ -195,9 +197,10 @@ class GlmOcrMTP(nn.Module, SupportsPP): if "scale" in name or "zero_point" in name: # Remapping the name of FP8 kv-scale or zero point. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: + remapped_name = maybe_remap_kv_scale_name(name, params_dict) + if remapped_name is None: continue + name = remapped_name for param_name, weight_name, shard_id in stacked_params_mapping: if weight_name not in name: diff --git a/vllm/model_executor/models/glmasr.py b/vllm/model_executor/models/glmasr.py index c9d0638e0c..219a7ce02d 100644 --- a/vllm/model_executor/models/glmasr.py +++ b/vllm/model_executor/models/glmasr.py @@ -11,7 +11,7 @@ from transformers.models.glmasr import GlmAsrConfig, GlmAsrProcessor from transformers.models.whisper import WhisperFeatureExtractor from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions from vllm.config.speech_to_text import SpeechToTextParams from vllm.distributed.parallel_state import get_tensor_model_parallel_world_size from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TokensPrompt @@ -710,6 +710,7 @@ class GlmAsrDummyInputsBuilder(BaseDummyInputsBuilder[GlmAsrProcessingInfo]): sampling_rate = feature_extractor.sampling_rate num_audios = mm_counts.get("audio", 0) audio_overrides = mm_options.get("audio") + assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) max_audio_len = getattr( self.info.get_hf_processor(), "max_audio_len", DEFAULT_MAX_AUDIO_LEN_S @@ -793,8 +794,13 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"]) ) processed_data["feature_attention_mask"] = mask - audio = mm_data.get("audio", []) - audio_list = [audio] if audio and not isinstance(audio, list) else audio + audio = mm_data.get("audio") + if audio is None: + audio_list: list[Any] = [] + elif isinstance(audio, list): + audio_list = audio + else: + audio_list = [audio] if audio_list: processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) @@ -878,6 +884,7 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"]) audio_embeds = out_mm_data.get("audio_embeds") if audio_embeds is not None: embed = audio_embeds[item_idx] + assert isinstance(embed, torch.Tensor) num_features = embed.shape[0] else: raise ValueError( @@ -1118,7 +1125,11 @@ class GlmAsrForConditionalGeneration( audio_token = cls._get_audio_token(model_config) if task_type == "translate": - full_lang_name_to = cls.supported_languages.get(to_language, to_language) + full_lang_name_to = ( + cls.supported_languages.get(to_language, to_language) + if to_language is not None + else "" + ) user_content = f"{audio_token}translate the speech to {full_lang_name_to}" elif task_type == "transcribe": user_content = ( diff --git a/vllm/model_executor/models/gpt_j.py b/vllm/model_executor/models/gpt_j.py index 8c74e6ad6f..04fe9ba549 100644 --- a/vllm/model_executor/models/gpt_j.py +++ b/vllm/model_executor/models/gpt_j.py @@ -227,6 +227,7 @@ class GPTJModel(nn.Module): else: hidden_states = self.embed_input_ids(input_ids) else: + assert intermediate_tensors is not None hidden_states = intermediate_tensors["hidden_states"] for layer in islice(self.h, self.start_layer, self.end_layer): hidden_states = layer(position_ids, hidden_states) diff --git a/vllm/model_executor/models/gpt_neox.py b/vllm/model_executor/models/gpt_neox.py index e360580b93..6e84eebe51 100644 --- a/vllm/model_executor/models/gpt_neox.py +++ b/vllm/model_executor/models/gpt_neox.py @@ -244,6 +244,7 @@ class GPTNeoXModel(nn.Module): else: hidden_states = self.embed_input_ids(input_ids) else: + assert intermediate_tensors is not None hidden_states = intermediate_tensors["hidden_states"] for layer in islice(self.layers, self.start_layer, self.end_layer): hidden_states = layer(position_ids, hidden_states) diff --git a/vllm/model_executor/models/gpt_oss.py b/vllm/model_executor/models/gpt_oss.py index 7f4de57f41..4887007564 100644 --- a/vllm/model_executor/models/gpt_oss.py +++ b/vllm/model_executor/models/gpt_oss.py @@ -79,6 +79,13 @@ _GPT_OSS_STREAMED_EXPERT_SUFFIX_TO_SHARD = { } +def _get_weight_loader(param: torch.Tensor) -> Callable[..., object]: + weight_loader = getattr(param, "weight_loader", default_weight_loader) + if not callable(weight_loader): + raise TypeError("weight_loader must be callable") + return weight_loader + + class OAIAttention(nn.Module): # Override to switch RoPE convention. gpt-oss uses NeoX (chunk halves); # privacy-filter and similar derivatives use GPT-J (interleaved pairs). @@ -273,6 +280,28 @@ class GptOssRoutedExperts(RoutedExperts): self._copy_to_expert(expert_data, loaded_weight) + @typing.overload + def weight_loader( + self, + param: torch.nn.Parameter, + loaded_weight: torch.Tensor, + weight_name: str, + shard_id: str, + expert_id: int, + return_success: typing.Literal[False], + ) -> None: ... + + @typing.overload + def weight_loader( + self, + param: torch.nn.Parameter, + loaded_weight: torch.Tensor, + weight_name: str, + shard_id: str, + expert_id: int, + return_success: typing.Literal[True], + ) -> bool: ... + def weight_loader( self, param: torch.nn.Parameter, @@ -283,14 +312,24 @@ class GptOssRoutedExperts(RoutedExperts): return_success: bool = False, ) -> bool | None: if shard_id not in ("gpt_oss_w13", "gpt_oss_w2"): - return super().weight_loader( + if return_success: + return super().weight_loader( + param, + loaded_weight, + weight_name, + shard_id, + expert_id, + True, + ) + super().weight_loader( param, loaded_weight, weight_name, shard_id, expert_id, - return_success, + False, ) + return None expert_id = self._map_global_expert_id_to_local_expert_id(expert_id) if expert_id == -1: @@ -518,7 +557,7 @@ class GptOssModel(nn.Module, EagleModelMixin): heads_per_rank: int, head_start: int, weights: Iterable[tuple[str, torch.Tensor]], - stacked_params_mapping: list[tuple[str, ...]], + stacked_params_mapping: list[tuple[str, str, str]], ) -> set[str]: params_dict = dict(self.named_parameters()) loaded_params: set[str] = set() @@ -564,7 +603,7 @@ class GptOssModel(nn.Module, EagleModelMixin): narrow_weight = weight[:, 2 * tp_rank_start : 2 * tp_rank_end, ...] param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader = _get_weight_loader(param) weight_loader( param, narrow_weight, @@ -586,7 +625,7 @@ class GptOssModel(nn.Module, EagleModelMixin): ] param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader = _get_weight_loader(param) weight_loader( param, narrow_weight, @@ -612,7 +651,7 @@ class GptOssModel(nn.Module, EagleModelMixin): narrow_weight = weight[:, 2 * tp_rank_start : 2 * tp_rank_end, ...] param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader = _get_weight_loader(param) weight_loader( param, narrow_weight, @@ -635,7 +674,7 @@ class GptOssModel(nn.Module, EagleModelMixin): narrow_weight = weight[..., tp_rank_start // 2 : tp_rank_end // 2] param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader = _get_weight_loader(param) weight_loader( param, narrow_weight, @@ -654,7 +693,7 @@ class GptOssModel(nn.Module, EagleModelMixin): narrow_weight = weight[:, 2 * tp_rank_start : 2 * tp_rank_end] param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader = _get_weight_loader(param) weight_loader( param, narrow_weight, @@ -667,7 +706,7 @@ class GptOssModel(nn.Module, EagleModelMixin): elif ".w2_bias" in name: # Handle MLP down projection bias param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader = _get_weight_loader(param) if use_ep: weight = weight[ep_rank_start:ep_rank_end, ...] else: @@ -691,7 +730,7 @@ class GptOssModel(nn.Module, EagleModelMixin): continue name = name.replace(weight_name, param_name) param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader = _get_weight_loader(param) if weight_loader == default_weight_loader: weight_loader(param, weight) else: @@ -702,7 +741,7 @@ class GptOssModel(nn.Module, EagleModelMixin): if name not in params_dict: continue param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader = _get_weight_loader(param) weight_loader(param, weight) loaded_params.add(name) return loaded_params @@ -714,7 +753,7 @@ class GptOssModel(nn.Module, EagleModelMixin): heads_per_rank: int, head_start: int, weights: Iterable[tuple[str, torch.Tensor]], - stacked_params_mapping: list[tuple[str, ...]], + stacked_params_mapping: list[tuple[str, str, str]], ) -> set[str]: params_dict = dict(self.named_parameters()) loaded_params: set[str] = set() @@ -820,6 +859,7 @@ class GptOssModel(nn.Module, EagleModelMixin): and expert_id is not None ): assert loaded_weight.numel() == 1 + assert fused_name is not None expert_data = params_dict[fused_name].data[expert_id] expert_data.copy_(loaded_weight) loaded_params.add(fused_name) @@ -901,6 +941,7 @@ class GptOssModel(nn.Module, EagleModelMixin): # fused gate_up_proj fused on disk, we cannot use the existing # weight loaders without added complexity, so just do the # direct load here. + assert fused_name is not None param = params_dict[fused_name] expert_data = param.data[expert_id] dim1 = sliced_weight.shape[0] @@ -1062,9 +1103,10 @@ class GptOssModel(nn.Module, EagleModelMixin): if name.endswith("scale"): # Remapping the name of FP8 kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: + remapped_name = maybe_remap_kv_scale_name(name, params_dict) + if remapped_name is None: continue + name = remapped_name param = params_dict[name] weight_loader = param.weight_loader @@ -1084,6 +1126,7 @@ class GptOssModel(nn.Module, EagleModelMixin): if weight_name not in name: continue + assert fused_name is not None param = params_dict[fused_name] # We should ask the weight loader to return success or not # here since otherwise we may skip experts with other @@ -1112,9 +1155,7 @@ class GptOssModel(nn.Module, EagleModelMixin): if name not in params_dict: continue param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) + weight_loader = _get_weight_loader(param) weight_loader(param, loaded_weight) loaded_params.add(name) @@ -1127,7 +1168,7 @@ class GptOssModel(nn.Module, EagleModelMixin): heads_per_rank: int, head_start: int, weights: Iterable[tuple[str, torch.Tensor]], - stacked_params_mapping: list[tuple[str, ...]], + stacked_params_mapping: list[tuple[str, str, str]], ) -> set[str]: params_dict = dict(self.named_parameters()) loaded_params: set[str] = set() @@ -1224,7 +1265,7 @@ class GptOssModel(nn.Module, EagleModelMixin): continue name = name.replace(weight_name, param_name) param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader = _get_weight_loader(param) if weight_loader == default_weight_loader: weight_loader(param, weight) else: @@ -1235,7 +1276,7 @@ class GptOssModel(nn.Module, EagleModelMixin): if name not in params_dict: continue param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader = _get_weight_loader(param) weight_loader(param, weight) loaded_params.add(name) return loaded_params @@ -1367,7 +1408,7 @@ class GptOssModel(nn.Module, EagleModelMixin): class GptOssForCausalLM( nn.Module, SupportsPP, SupportsEagle, SupportsEagle3, SupportsLoRA ): - is_3d_moe_weight: bool = True + is_3d_moe_weight: typing.ClassVar[bool] = True packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]} hf_to_vllm_mapper = WeightsMapper( diff --git a/vllm/model_executor/models/granite4_vision.py b/vllm/model_executor/models/granite4_vision.py index e82a42bf87..50b0932bda 100644 --- a/vllm/model_executor/models/granite4_vision.py +++ b/vllm/model_executor/models/granite4_vision.py @@ -130,6 +130,7 @@ class WindowQFormerDownsampler(nn.Module): self.dropout = nn.Dropout(config.projector_dropout) + self.downsampler: SpatialOffsetDownsampler | InterpolateDownsampler if spatial_offset is not None: self.downsampler = SpatialOffsetDownsampler(config, offset=spatial_offset) else: @@ -843,6 +844,7 @@ class Granite4VisionForConditionalGeneration( # (possibly a chunk slice from the framework's encoder cache). # Concatenate along token dim → (total_mm_tokens, lm_h * num_levels). N, lm_h = inputs_embeds.shape + assert multimodal_embeddings is not None all_packed = torch.cat( [t.to(dtype=inputs_embeds.dtype) for t in multimodal_embeddings], dim=0, diff --git a/vllm/model_executor/models/granite_speech.py b/vllm/model_executor/models/granite_speech.py index 9233e772d7..0fac643999 100644 --- a/vllm/model_executor/models/granite_speech.py +++ b/vllm/model_executor/models/granite_speech.py @@ -34,7 +34,7 @@ from torch import nn from transformers import BatchFeature, PretrainedConfig from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions from vllm.config.speech_to_text import SpeechToTextParams from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear @@ -166,7 +166,9 @@ class GraniteSpeechMultiModalProcessor( def get_replacement(item_idx: int): audios = mm_items.get_items("audio", AudioProcessorItems) audio = audios.get(item_idx) - audio_length = audio.shape[-1] + if audio is None: + raise ValueError(f"Missing audio item {item_idx}") + audio_length = len(audio) if isinstance(audio, list) else audio.shape[-1] num_projector_features = feature_extractor._get_num_audio_features( [audio_length] )[0] @@ -225,6 +227,7 @@ class GraniteSpeechDummyInputsBuilder( ) -> MultiModalDataDict: num_audios = mm_counts.get("audio", 0) audio_overrides = mm_options.get("audio") + assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) return { "audio": self._get_dummy_audios( @@ -659,6 +662,9 @@ class GraniteSpeechForConditionalGeneration( if input_features is None: return None + if not isinstance(audio_embed_sizes, torch.Tensor): + raise ValueError("audio_embed_sizes must be a tensor") + # If we have a batch of variable feature length audio clips, we need # to mask the features; usually we would get an input_features_mask # from the processor, but we handle rebuilding it here since @@ -879,7 +885,11 @@ class GraniteSpeechForConditionalGeneration( audio_tok = cls.get_placeholder_str("audio", 0) if task_type == "translate": - full_lang_name_to = cls.supported_languages.get(to_language, to_language) + full_lang_name_to = ( + cls.supported_languages.get(to_language, to_language) + if to_language is not None + else "" + ) user_prompt = f"{audio_tok}translate the speech to {full_lang_name_to}" # noqa: E501 elif task_type == "transcribe": user_prompt = ( diff --git a/vllm/model_executor/models/granitemoehybrid.py b/vllm/model_executor/models/granitemoehybrid.py index aa21c23f1e..52b3a72a74 100644 --- a/vllm/model_executor/models/granitemoehybrid.py +++ b/vllm/model_executor/models/granitemoehybrid.py @@ -270,6 +270,7 @@ class GraniteMoeHybridAttention(nn.Module): prefix=f"{prefix}.o_proj", ) + self.rotary_emb: nn.Module | None if config.position_embedding_type == "rope": self.rotary_emb = get_rope( self.head_dim, @@ -454,7 +455,9 @@ class GraniteMoeHybridModel(nn.Module): # Skip layers on other devices. if not is_pp_missing_parameter(n, self): param = params_dict[n] - weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader = getattr(param, "weight_loader", None) + if not callable(weight_loader): + raise TypeError(f"{n} requires a shard-aware weight loader") weight_loader(param, p, shard_id) loaded_params.add(n) @@ -462,7 +465,9 @@ class GraniteMoeHybridModel(nn.Module): if n not in params_dict: return param = params_dict[n] - weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader = getattr(param, "weight_loader", None) + if not callable(weight_loader): + raise TypeError(f"{n} requires an expert-aware weight loader") weight_loader(param, p, name, shard_id=shard_id, expert_id=expert_id) loaded_params.add(n)