diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index 0feac74a52..b5cbc9e136 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -111,10 +111,7 @@ SEPARATE_GROUPS = [ "tests/v1/worker", ] -# TODO(woosuk): Include the code from Megatron and HuggingFace. EXCLUDE = [ - r"vllm/model_executor/models/[cC]", - r"vllm/model_executor/models/[dD]", r"vllm/model_executor/models/[eE]", r"vllm/model_executor/models/[fF]", r"vllm/model_executor/models/[gG]", diff --git a/vllm/model_executor/models/clip.py b/vllm/model_executor/models/clip.py index 8919d65de3..c37e2e6214 100644 --- a/vllm/model_executor/models/clip.py +++ b/vllm/model_executor/models/clip.py @@ -142,6 +142,7 @@ class CLIPProcessingInfo(BaseProcessingInfo): pooler_config = self.ctx.model_config.pooler_config assert pooler_config is not None + assert pooler_config.seq_pooling_type is not None return get_num_selected_vision_tokens( vision_encoder_info.get_num_image_tokens( @@ -854,6 +855,7 @@ class CLIPEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant): feature_select_strategy: VisionFeatureSelectStrategy | None = None, ) -> torch.Tensor: if feature_select_strategy is None: + assert self.pooler_config.seq_pooling_type is not None feature_select_strategy = _get_vision_feature_select_strategy( self.pooler_config.seq_pooling_type ) @@ -961,6 +963,7 @@ class CLIPEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant): raise RuntimeError("PP is not supported for this model") # Multimodal inputs + assert inputs_embeds is not None if not self._is_text_input: return inputs_embeds diff --git a/vllm/model_executor/models/cohere2_moe.py b/vllm/model_executor/models/cohere2_moe.py index 3365ab06fc..12291fb142 100644 --- a/vllm/model_executor/models/cohere2_moe.py +++ b/vllm/model_executor/models/cohere2_moe.py @@ -178,6 +178,7 @@ class Cohere2MoeAttention(nn.Module): self.max_position_embeddings = getattr( config, "model_max_length", None ) or getattr(config, "max_position_embeddings", 8192) + assert isinstance(self.max_position_embeddings, int) self.qkv_proj = QKVParallelLinear( self.hidden_size, self.head_dim, @@ -283,6 +284,7 @@ class Cohere2Moe(nn.Module): prefix=f"{prefix}.gate", ) + self.shared_experts: Cohere2MoeMLP | None if hasattr(config, "num_shared_experts") and config.num_shared_experts > 0: self.shared_experts = Cohere2MoeMLP( config=config, diff --git a/vllm/model_executor/models/cohere2_vision.py b/vllm/model_executor/models/cohere2_vision.py index 302619a8db..b58df12d0d 100644 --- a/vllm/model_executor/models/cohere2_vision.py +++ b/vllm/model_executor/models/cohere2_vision.py @@ -416,7 +416,7 @@ class Cohere2VisionForConditionalGeneration( ) def _patch_quant_config( - self, config: PretrainedConfig, quant_config: QuantizationConfig + self, config: PretrainedConfig, quant_config: QuantizationConfig | None ): # the awq models from OpenGVLab missing `modules_to_not_convert` # patch the quant_config to add `modules_to_not_convert` back diff --git a/vllm/model_executor/models/cohere_asr.py b/vllm/model_executor/models/cohere_asr.py index da74404139..b124f30d52 100644 --- a/vllm/model_executor/models/cohere_asr.py +++ b/vllm/model_executor/models/cohere_asr.py @@ -55,6 +55,7 @@ from vllm.transformers_utils.processors.cohere_asr import ( CohereASRFeatureExtractor, CohereASRProcessor, ) +from vllm.utils.collection_utils import is_list_of from vllm.v1.attention.backend import ( AttentionType, ) @@ -183,6 +184,7 @@ class CohereASRAttention(nn.Module): def forward( self, hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor | None = None, ) -> torch.Tensor: qkv, _ = self.qkv_proj(hidden_states) q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) @@ -241,7 +243,7 @@ class CohereASRCrossAttention(CohereASRAttention): def forward( self, hidden_states: torch.Tensor, - encoder_hidden_states: torch.Tensor | None, + encoder_hidden_states: torch.Tensor | None = None, ) -> torch.Tensor: q, _ = self.q_proj(hidden_states) @@ -837,7 +839,7 @@ class CausalConv1D(nn.Conv1d): out_channels: int, kernel_size: int, stride: int = 1, - padding: str | int = 0, + padding: str | int | list[int] | None = 0, dilation: int = 1, groups: int = 1, bias: bool = True, @@ -845,6 +847,8 @@ class CausalConv1D(nn.Conv1d): device=None, dtype=None, ) -> None: + self._left_padding: int + self._right_padding: int if padding is None: self._left_padding = kernel_size - 1 self._right_padding = stride - 1 @@ -903,7 +907,7 @@ class ConformerConvolution(nn.Module): d_model: int, kernel_size: int, norm_type: str = "batch_norm", - conv_context_size: int | None = None, + conv_context_size: int | list[int] | None = None, pointwise_activation: str = "glu_", use_bias: bool = True, ) -> None: @@ -1158,6 +1162,7 @@ class RelPositionMultiHeadAttention(CohereASRMultiHeadAttention): q, k, v = self.forward_qkv(query, key, value) q = q.transpose(1, 2) # (batch, time1, head, d_k) + assert pos_emb is not None n_batch_pos = pos_emb.size(0) p = self.linear_pos(pos_emb).view(n_batch_pos, -1, self.h, self.d_k) p = p.transpose(1, 2) # (batch, head, time1, d_k) @@ -1213,7 +1218,7 @@ class ConformerLayer(torch.nn.Module): n_heads: int = 4, conv_kernel_size: int = 31, conv_norm_type: str = "batch_norm", - conv_context_size: int | None = None, + conv_context_size: int | list[int] | None = None, pos_bias_u: nn.Parameter | torch.Tensor | None = None, pos_bias_v: nn.Parameter | torch.Tensor | None = None, att_context_size: list[int] | None = None, @@ -1382,6 +1387,7 @@ class ConformerEncoder(nn.Module): conv_kernel_size=conv_kernel_size, ) + self.xscale: float | None if xscaling: self.xscale = math.sqrt(d_model) else: @@ -1633,9 +1639,12 @@ class ConformerEncoder(nn.Module): ) -> tuple[list[list[int]], list[int], list[float], list[int]]: # convert att_context_size to a standard list of lists if att_context_size: - att_context_size_all = list(att_context_size) - if isinstance(att_context_size_all[0], int): - att_context_size_all = [att_context_size_all] + if is_list_of(att_context_size, int, check="all"): + att_context_size_all = [att_context_size] + elif is_list_of(att_context_size, list, check="all"): + att_context_size_all = att_context_size + else: + raise ValueError("att_context_size cannot mix nested and flat values") for i, att_cs in enumerate(att_context_size_all): if att_context_style == "chunked_limited": if att_cs[0] > 0 and att_cs[0] % (att_cs[1] + 1) > 0: @@ -1684,6 +1693,7 @@ class ConformerEncoder(nn.Module): if conv_context_size == "causal": conv_context_size = [conv_kernel_size - 1, 0] else: + assert isinstance(conv_context_size, list) total = conv_context_size[0] + conv_context_size[1] + 1 if total != conv_kernel_size: raise ValueError( @@ -1940,7 +1950,7 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing ): if mm_data: feature_extractor = self.info.get_feature_extractor(**mm_kwargs) - mm_data = dict(audio=mm_data.pop("audios")) + mm_data = dict(audio=dict(mm_data).pop("audios")) mm_kwargs = dict( **mm_kwargs, sampling_rate=feature_extractor.sampling_rate, @@ -2045,7 +2055,6 @@ class CohereAsrForConditionalGeneration( # (which is different from "_") and encode("▁ABC") ignores the first token # so the prompt_text is unreliable. However, prompt_token_ids can be used # to get prompt_text but it wont have the first token "▁". - prompt_text = None prompt_token_ids = cls._get_default_prompt_token_ids( tokenizer, model_config, @@ -2053,7 +2062,6 @@ class CohereAsrForConditionalGeneration( ) return TokensPrompt( - prompt=prompt_text, prompt_token_ids=prompt_token_ids, multi_modal_data={"audio": (audio, stt_config.sample_rate)}, ) @@ -2199,7 +2207,9 @@ class CohereAsrForConditionalGeneration( audio_input, seq_lens = self._parse_and_validate_audio_input(**kwargs) if hasattr(audio_input, "input_features"): - out = self.model.get_encoder_outputs(audio_input["input_features"]) + out = self.model.get_encoder_outputs( + audio_input["input_features"], seq_lens + ) else: out = self.model.get_encoder_outputs(audio_input, seq_lens) @@ -2219,12 +2229,17 @@ class CohereAsrForConditionalGeneration( f"Incorrect type of audio features. Got type: {type(input_features)}" ) + if not isinstance(length, torch.Tensor): + raise ValueError("Audio feature lengths must be a tensor.") + if isinstance(input_features, torch.Tensor): seq_lens = length.reshape(-1) else: + if not is_list_of(input_features, torch.Tensor, check="all"): + raise ValueError("All audio features must be tensors.") input_features = [ - feat.to(self.dtype).squeeze(0).transpose(1, 0) - for feat in input_features + feature.to(self.dtype).squeeze(0).transpose(1, 0) + for feature in input_features ] seq_lens = length.reshape(-1) input_features = torch.nn.utils.rnn.pad_sequence( diff --git a/vllm/model_executor/models/cohere_eagle.py b/vllm/model_executor/models/cohere_eagle.py index 64ec0d6dd5..519d1164b0 100644 --- a/vllm/model_executor/models/cohere_eagle.py +++ b/vllm/model_executor/models/cohere_eagle.py @@ -58,6 +58,7 @@ class CohereEagleModel(nn.Module): start_layer_id: int = 0, ) -> None: super().__init__() + assert vllm_config.speculative_config is not None self.config = vllm_config.speculative_config.draft_model_config.hf_config self.quant_config = get_draft_quant_config(vllm_config) @@ -137,6 +138,7 @@ class CohereEagleModel(nn.Module): class EagleCohereForCausalLM(CohereForCausalLM): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): nn.Module.__init__(self) + assert vllm_config.speculative_config is not None self.config = vllm_config.speculative_config.draft_model_config.hf_config # Flags checked by the speculative proposer to decide whether to share # embed_tokens / lm_head with the target model. Cohere EAGLE checkpoints @@ -160,7 +162,7 @@ class EagleCohereForCausalLM(CohereForCausalLM): def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.model.embed_input_ids(input_ids) - def forward( + def forward( # type: ignore[override] self, input_ids: torch.Tensor, positions: torch.Tensor, diff --git a/vllm/model_executor/models/colmodernvbert.py b/vllm/model_executor/models/colmodernvbert.py index b16bb4e221..798b87b77b 100644 --- a/vllm/model_executor/models/colmodernvbert.py +++ b/vllm/model_executor/models/colmodernvbert.py @@ -33,6 +33,7 @@ from vllm.multimodal.processing import ( PromptUpdate, ) from vllm.sequence import IntermediateTensors +from vllm.tokenizers.hf import HfTokenizer from vllm.transformers_utils.configs.colmodernvbert import ColModernVBertConfig from .interfaces import ( @@ -161,6 +162,7 @@ class ColModernVBertMultiModalProcessor( tok_kwargs: Mapping[str, object], ) -> BatchFeature: tokenizer = self.info.get_tokenizer() + assert isinstance(tokenizer, HfTokenizer) text_encoding = tokenizer( prompt, return_tensors="pt", diff --git a/vllm/model_executor/models/colpali.py b/vllm/model_executor/models/colpali.py index 18317c0aad..7948da01ba 100644 --- a/vllm/model_executor/models/colpali.py +++ b/vllm/model_executor/models/colpali.py @@ -25,6 +25,7 @@ from transformers import BatchFeature, PaliGemmaProcessor from vllm.config import VllmConfig from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed +from vllm.model_executor.layers.pooler.tokwise.heads import TokenPoolerHead from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.multimodal import MULTIMODAL_REGISTRY @@ -232,7 +233,9 @@ class ColPaliModel( loaded.add(f"custom_text_proj.{param_name}") # Update pooler projector for the lazy-creation path - self.pooler.head.projector = self.custom_text_proj + head = self.pooler.head + assert isinstance(head, TokenPoolerHead) + head.projector = self.custom_text_proj # Mark pooler projector params as loaded if hasattr(self, "pooler") and hasattr(self.pooler, "head"): diff --git a/vllm/model_executor/models/commandr.py b/vllm/model_executor/models/commandr.py index 813e0a2f5f..3c755c65e1 100644 --- a/vllm/model_executor/models/commandr.py +++ b/vllm/model_executor/models/commandr.py @@ -154,6 +154,7 @@ class CohereAttention(nn.Module): self.max_position_embeddings = getattr( config, "model_max_length", None ) or getattr(config, "max_position_embeddings", 8192) + assert isinstance(self.max_position_embeddings, int) self.use_qk_norm = getattr(config, "use_qk_norm", False) self.qkv_proj = QKVParallelLinear( self.hidden_size, diff --git a/vllm/model_executor/models/config.py b/vllm/model_executor/models/config.py index c507b28664..d294943318 100644 --- a/vllm/model_executor/models/config.py +++ b/vllm/model_executor/models/config.py @@ -9,6 +9,7 @@ if TYPE_CHECKING: from transformers import PretrainedConfig from vllm.config import CacheConfig, ModelConfig, VllmConfig + from vllm.config.cache import MambaDType logger = init_logger(__name__) @@ -452,6 +453,7 @@ class JambaForSequenceClassificationConfig(VerifyAndUpdateConfig): @staticmethod def verify_and_update_model_config(model_config: "ModelConfig") -> None: pooler_config = model_config.pooler_config + assert pooler_config is not None if pooler_config.use_activation is None: pooler_config.use_activation = False @@ -518,6 +520,7 @@ class JinaVLForSequenceClassificationConfig(VerifyAndUpdateConfig): config = model_config.hf_config config.num_labels = 1 pooler_config = model_config.pooler_config + assert pooler_config is not None if pooler_config.logit_mean is None: pooler_config.logit_mean = 2.65 @@ -540,6 +543,7 @@ class LlamaBidirectionalConfig(VerifyAndUpdateConfig): if pooling_type is None: raise ValueError(f"pool_type {hf_config.pooling!r} not supported") + assert model_config.pooler_config is not None model_config.pooler_config.seq_pooling_type = pooling_type @@ -572,10 +576,13 @@ class LlamaNemotronVLConfig(VerifyAndUpdateConfig): if pooling is None and hasattr(hf_config, "llm_config"): pooling = getattr(hf_config.llm_config, "pooling", "avg") + assert isinstance(pooling, str) + pooling_type = pooling_type_map.get(pooling) if pooling_type is None: raise ValueError(f"pool_type {pooling!r} not supported") + assert model_config.pooler_config is not None model_config.pooler_config.seq_pooling_type = pooling_type @@ -631,7 +638,7 @@ class MambaModelConfig(VerifyAndUpdateConfig): class NemotronHForCausalLMConfig(VerifyAndUpdateConfig): - DEFAULT_MAMBA_SSM_CACHE_DTYPE = "float32" + DEFAULT_MAMBA_SSM_CACHE_DTYPE: "MambaDType" = "float32" """Only `float32` is known to have no accuracy issues by default.""" @classmethod @@ -643,7 +650,7 @@ class NemotronHForCausalLMConfig(VerifyAndUpdateConfig): `float32` if not specified. """ if cache_config.mamba_ssm_cache_dtype == "auto": - mamba_ssm_cache_dtype = getattr( + mamba_ssm_cache_dtype: MambaDType = getattr( hf_config, "mamba_ssm_cache_dtype", cls.DEFAULT_MAMBA_SSM_CACHE_DTYPE ) logger.info( @@ -729,6 +736,7 @@ class Qwen2ForProcessRewardModelConfig(VerifyAndUpdateConfig): @staticmethod def verify_and_update_model_config(model_config: "ModelConfig") -> None: pooler_config = model_config.pooler_config + assert pooler_config is not None if pooler_config.step_tag_id is None: pooler_config.step_tag_id = 151651 @@ -738,6 +746,7 @@ class Qwen2ForRewardModelConfig(VerifyAndUpdateConfig): @staticmethod def verify_and_update_model_config(model_config: "ModelConfig") -> None: pooler_config = model_config.pooler_config + assert pooler_config is not None if pooler_config.use_activation is None: pooler_config.use_activation = False diff --git a/vllm/model_executor/models/cosmos3_edge.py b/vllm/model_executor/models/cosmos3_edge.py index 62ab7922c6..82592751ab 100644 --- a/vllm/model_executor/models/cosmos3_edge.py +++ b/vllm/model_executor/models/cosmos3_edge.py @@ -66,7 +66,7 @@ class Cosmos3EdgeVisionEncoder(Siglip2VisionTransformer): def dtype(self) -> torch.dtype: return self.embeddings.patch_embedding.weight.dtype - def forward( + def encode( self, pixel_values: torch.Tensor, grid_thw: torch.Tensor, @@ -253,7 +253,7 @@ class Cosmos3EdgeVisionModel(nn.Module): grid_thw: torch.Tensor | list[list[int]], ) -> torch.Tensor: grid_thw = torch.as_tensor(grid_thw, dtype=torch.int64, device="cpu") - image_embeds = self.encoder(pixel_values.type(self.dtype), grid_thw=grid_thw) + image_embeds = self.encoder.encode(pixel_values.type(self.dtype), grid_thw) image_embeds = patch_merging_by_param( image_embeds, grid_thw, @@ -315,11 +315,10 @@ class Cosmos3EdgeAttention(NemotronHAttention): rope_parameters=config.rope_parameters, ) - def forward( + def forward_with_positions( self, positions: torch.Tensor, hidden_states: torch.Tensor, - **kwargs, ) -> torch.Tensor: qkv, _ = self.qkv_proj(hidden_states) q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) @@ -367,7 +366,7 @@ class Cosmos3EdgeAttentionDecoderLayer(nn.Module): hidden_states = self.norm(hidden_states) else: hidden_states, residual = self.norm(hidden_states, residual) - hidden_states = self.mixer(positions=positions, hidden_states=hidden_states) + hidden_states = self.mixer.forward_with_positions(positions, hidden_states) return hidden_states, residual @@ -562,6 +561,7 @@ class Cosmos3EdgeForConditionalGeneration( config = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config multimodal_config = vllm_config.model_config.multimodal_config + assert multimodal_config is not None self.config = config self.multimodal_config = multimodal_config @@ -619,6 +619,8 @@ class Cosmos3EdgeForConditionalGeneration( image_grid_thw=image_grid_thw, ) + raise AssertionError + def _parse_and_validate_video_input( self, **kwargs: object ) -> Qwen2_5_VLVideoInputs | None: @@ -645,6 +647,8 @@ class Cosmos3EdgeForConditionalGeneration( video_grid_thw=video_grid_thw, ) + raise AssertionError + def _process_image_input( self, image_input: Qwen2_5_VLImageInputs ) -> tuple[torch.Tensor, ...]: @@ -684,7 +688,7 @@ class Cosmos3EdgeForConditionalGeneration( return self._get_image_features(pixel_values_videos, grid_thw) def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: - modalities = {} + modalities: dict = {} image_input = self._parse_and_validate_image_input(**kwargs) if image_input is not None: modalities["image"] = image_input diff --git a/vllm/model_executor/models/dbrx.py b/vllm/model_executor/models/dbrx.py index 7f1ddc442a..0f5ffeff03 100644 --- a/vllm/model_executor/models/dbrx.py +++ b/vllm/model_executor/models/dbrx.py @@ -3,6 +3,7 @@ from collections.abc import Iterable from itertools import islice +from typing import Any import torch import torch.nn as nn @@ -93,8 +94,11 @@ class DbrxExperts(RoutedExperts): param: nn.Parameter, loaded_weight: torch.Tensor, weight_name: str, - param_name: str, - ): + shard_id: str, + expert_id: int | None = None, + return_success: Any = False, + ) -> Any: + param_name = shard_id tp_rank = get_tensor_model_parallel_rank() param_data = param.data shard_size = self.intermediate_size @@ -134,6 +138,7 @@ class DbrxExperts(RoutedExperts): param_data[:] = loaded_weight[:, :, shard] else: param_data[:] = loaded_weight + return True if return_success else None class DbrxMoE(nn.Module): @@ -409,9 +414,10 @@ class DbrxModel(nn.Module): if is_pp_missing_parameter(name, self): continue # Remapping the name of FP8 kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: + remapped_name = maybe_remap_kv_scale_name(name, params_dict) + if remapped_name is None: continue + name = remapped_name param = params_dict[name] weight_loader = getattr(param, "weight_loader", default_weight_loader) weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/deepencoder.py b/vllm/model_executor/models/deepencoder.py index fffd9382dd..1c786ea85a 100644 --- a/vllm/model_executor/models/deepencoder.py +++ b/vllm/model_executor/models/deepencoder.py @@ -10,7 +10,7 @@ # This source code is licensed under the license found in the # LICENSE file in the root directory of this source tree. import math -from collections.abc import Iterable +from collections.abc import Callable, Iterable from functools import partial import torch @@ -73,7 +73,7 @@ class ImageEncoderViT(nn.Module): mlp_ratio: float = 4.0, out_chans: int = 256, qkv_bias: bool = True, - norm_layer: type[nn.Module] = nn.LayerNorm, + norm_layer: Callable[[int], nn.Module] = nn.LayerNorm, act_layer: type[nn.Module] = nn.GELU, use_abs_pos: bool = True, use_rel_pos: bool = False, @@ -205,7 +205,7 @@ class Block(nn.Module): num_heads: int, mlp_ratio: float = 4.0, qkv_bias: bool = True, - norm_layer: type[nn.Module] = nn.LayerNorm, + norm_layer: Callable[[int], nn.Module] = nn.LayerNorm, act_layer: type[nn.Module] = nn.GELU, use_rel_pos: bool = False, rel_pos_zero_init: bool = True, @@ -326,6 +326,7 @@ class RelPosAttention(PluggableLayer): v = v.view(B, self.num_heads, H * W, -1) if self.use_rel_pos: + assert rel_h is not None and rel_w is not None rel_h = rel_h.view( B, self.num_heads, rel_h.size(1), rel_h.size(2), rel_h.size(3) ) diff --git a/vllm/model_executor/models/deepencoder2.py b/vllm/model_executor/models/deepencoder2.py index 9e251c39b2..432b4eb56d 100644 --- a/vllm/model_executor/models/deepencoder2.py +++ b/vllm/model_executor/models/deepencoder2.py @@ -15,6 +15,7 @@ from functools import lru_cache import torch import torch.nn as nn import transformers +from transformers.models.qwen2.modeling_qwen2 import Qwen2Model from vllm.model_executor.custom_op import PluggableLayer @@ -49,7 +50,6 @@ class CustomQwen2Decoder(PluggableLayer): super().__init__() # load - Qwen2Model = transformers.models.qwen2.modeling_qwen2.Qwen2Model Qwen2Config = transformers.Qwen2Config # config @@ -70,11 +70,11 @@ class CustomQwen2Decoder(PluggableLayer): ) # - self.model = self._create_custom_model(Qwen2Model, config) + self.model = self._create_custom_model(config) del self.model.embed_tokens - def _create_custom_model(self, Qwen2Model, config): + def _create_custom_model(self, config): """Qwen2Model""" class CustomQwen2ModelInner(Qwen2Model): diff --git a/vllm/model_executor/models/deepseek_eagle.py b/vllm/model_executor/models/deepseek_eagle.py index 76e90e3276..91e158fa5a 100644 --- a/vllm/model_executor/models/deepseek_eagle.py +++ b/vllm/model_executor/models/deepseek_eagle.py @@ -39,6 +39,7 @@ class DeepseekV2Model(nn.Module): start_layer_id: int = 0, ) -> None: super().__init__() + assert vllm_config.speculative_config is not None self.config = vllm_config.speculative_config.draft_model_config.hf_config quant_config = vllm_config.quant_config self.vocab_size = self.config.vocab_size @@ -155,7 +156,7 @@ class DeepseekV2Model(nn.Module): break else: for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping + param_name, weight_name, expert_id, expert_shard_id = mapping if weight_name not in name: continue name = name.replace(weight_name, param_name) @@ -166,7 +167,7 @@ class DeepseekV2Model(nn.Module): param, loaded_weight, name, - shard_id=shard_id, + shard_id=expert_shard_id, expert_id=expert_id, ) break @@ -176,9 +177,10 @@ class DeepseekV2Model(nn.Module): continue # Remapping the name of FP8 kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: + remapped_name = maybe_remap_kv_scale_name(name, params_dict) + if remapped_name is None: continue + name = remapped_name param = params_dict[name] weight_loader = getattr( @@ -192,6 +194,7 @@ class DeepseekV2Model(nn.Module): class EagleDeepseekV3ForCausalLM(DeepseekV3ForCausalLM): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): nn.Module.__init__(self) + assert vllm_config.speculative_config is not None self.config = vllm_config.speculative_config.draft_model_config.hf_config quant_config = vllm_config.quant_config target_layer_num = vllm_config.model_config.get_num_layers( @@ -222,7 +225,7 @@ class EagleDeepseekV3ForCausalLM(DeepseekV3ForCausalLM): def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.model.embed_input_ids(input_ids) - def forward( + def forward( # type: ignore[override] self, input_ids: torch.Tensor, positions: torch.Tensor, diff --git a/vllm/model_executor/models/deepseek_eagle3.py b/vllm/model_executor/models/deepseek_eagle3.py index faa2c26d9d..52f476adbc 100644 --- a/vllm/model_executor/models/deepseek_eagle3.py +++ b/vllm/model_executor/models/deepseek_eagle3.py @@ -159,6 +159,7 @@ class DeepseekV2Eagle3Model(nn.Module): prefix: str = "", ) -> None: super().__init__() + assert vllm_config.speculative_config is not None self.config = vllm_config.speculative_config.draft_model_config.hf_config self.vocab_size = self.config.vocab_size @@ -276,6 +277,7 @@ class Eagle3DeepseekV2ForCausalLM(LocalArgmaxMixin, DeepseekV2ForCausalLM): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): nn.Module.__init__(self) + assert vllm_config.speculative_config is not None self.config = vllm_config.speculative_config.draft_model_config.hf_config # Ensure draft_vocab_size is set @@ -318,7 +320,7 @@ class Eagle3DeepseekV2ForCausalLM(LocalArgmaxMixin, DeepseekV2ForCausalLM): ) -> torch.Tensor: return self.model.embed_input_ids(input_ids) - def forward( + def forward( # type: ignore[override] self, input_ids: torch.Tensor, positions: torch.Tensor, diff --git a/vllm/model_executor/models/deepseek_mtp.py b/vllm/model_executor/models/deepseek_mtp.py index 35cfce3cb9..7fc78f13ff 100644 --- a/vllm/model_executor/models/deepseek_mtp.py +++ b/vllm/model_executor/models/deepseek_mtp.py @@ -68,6 +68,7 @@ class DeepSeekMultiTokenPredictorLayer(nn.Module): def __init__(self, vllm_config: VllmConfig, prefix: str) -> None: super().__init__() + assert vllm_config.speculative_config is not None config = vllm_config.speculative_config.draft_model_config.hf_config self.config = config quant_config = vllm_config.quant_config @@ -423,7 +424,7 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts): # with expert_id. is_expert_weight = False for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping + param_name, weight_name, expert_id, expert_shard_id = mapping if weight_name not in chunk_name: continue @@ -446,7 +447,7 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts): param, weight_to_load, name_mapped, - shard_id=shard_id, + shard_id=expert_shard_id, expert_id=expert_id, return_success=True, ) @@ -467,9 +468,10 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts): if name.endswith(".bias") and name not in params_dict: continue - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: + remapped_name = maybe_remap_kv_scale_name(name, params_dict) + if remapped_name is None: continue + name = remapped_name # According to DeepSeek-V3 Technical Report, MTP modules # shares embedding layer. We only load the first weights. diff --git a/vllm/model_executor/models/deepseek_ocr.py b/vllm/model_executor/models/deepseek_ocr.py index 58cedbb0f9..ead643a7f8 100644 --- a/vllm/model_executor/models/deepseek_ocr.py +++ b/vllm/model_executor/models/deepseek_ocr.py @@ -49,6 +49,7 @@ from vllm.multimodal.processing import ( from vllm.sampling_params import SamplingParams from vllm.sequence import IntermediateTensors from vllm.tokenizers import cached_tokenizer_from_config +from vllm.tokenizers.hf import HfTokenizer from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config from vllm.transformers_utils.processors.deepseek_ocr import ( BASE_SIZE, @@ -184,6 +185,7 @@ class NGramPerReqLogitsProcessor(AdapterLogitsProcessor): ) if ngram_size is None: return None + assert isinstance(ngram_size, int) and isinstance(window_size, int) whitelist_token_ids = set(whitelist_token_ids) if whitelist_token_ids else None return NoRepeatNGramLogitsProcessor( @@ -301,6 +303,7 @@ class DeepseekOCRMultiModalProcessor( else: tokenizer = self.info.get_tokenizer() + assert isinstance(tokenizer, HfTokenizer) processed_outputs = tokenizer( prompt, add_special_tokens=True, return_tensors="pt" ) @@ -342,6 +345,7 @@ class DeepseekOCRMultiModalProcessor( if isinstance(images, ImageEmbeddingItems): num_image_tokens = images.get_feature_size(item_idx) else: + assert isinstance(images, ImageProcessorItems) size = images.get_image_size(item_idx) num_image_tokens = self.info.get_num_image_tokens( @@ -459,7 +463,11 @@ class DeepseekOCRForCausalLM( images_spatial_crop = kwargs.pop("images_spatial_crop", None) images_crop = kwargs.pop("images_crop", None) - if pixel_values is None or torch.sum(pixel_values).item() == 0: + if pixel_values is None: + return None + assert isinstance(pixel_values, torch.Tensor) + assert images_crop is None or isinstance(images_crop, torch.Tensor) + if torch.sum(pixel_values).item() == 0: return None # Use actual tensor spatial dim instead of hardcoded @@ -675,7 +683,8 @@ class DeepseekOCRForCausalLM( num_input_tokens = global_input_side**2 if is_tiled: - num_patches = image_spatial_crop.prod(dim=-1) + assert image_spatial_crop is not None + num_patches = int(image_spatial_crop.prod().item()) num_input_tokens += num_patches * (local_input_side**2) global_output_token = self.global_image_output_token diff --git a/vllm/model_executor/models/deepseek_ocr2.py b/vllm/model_executor/models/deepseek_ocr2.py index 70f50ea7d4..f96fb9c146 100644 --- a/vllm/model_executor/models/deepseek_ocr2.py +++ b/vllm/model_executor/models/deepseek_ocr2.py @@ -47,6 +47,7 @@ from vllm.multimodal.processing import ( ) from vllm.sequence import IntermediateTensors from vllm.tokenizers import cached_tokenizer_from_config +from vllm.tokenizers.hf import HfTokenizer from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config from vllm.transformers_utils.processors.deepseek_ocr import ( BASE_SIZE, @@ -173,6 +174,7 @@ class DeepseekOCR2MultiModalProcessor( else: tokenizer = self.info.get_tokenizer() + assert isinstance(tokenizer, HfTokenizer) processed_outputs = tokenizer( prompt, add_special_tokens=True, return_tensors="pt" ) @@ -214,6 +216,7 @@ class DeepseekOCR2MultiModalProcessor( if isinstance(images, ImageEmbeddingItems): num_image_tokens = images.get_feature_size(item_idx) else: + assert isinstance(images, ImageProcessorItems) size = images.get_image_size(item_idx) num_image_tokens = self.info.get_num_image_tokens( @@ -284,7 +287,7 @@ class DeepseekOCR2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, Support patch_size=16, qkv_bias=True, use_rel_pos=True, - global_attn_indexes=[2, 5, 8, 11], + global_attn_indexes=(2, 5, 8, 11), window_size=14, out_chans=256, last_conv_output=896, diff --git a/vllm/model_executor/models/deepseek_v2.py b/vllm/model_executor/models/deepseek_v2.py index 254763706c..81e941d284 100644 --- a/vllm/model_executor/models/deepseek_v2.py +++ b/vllm/model_executor/models/deepseek_v2.py @@ -626,7 +626,7 @@ class DeepseekV32IndexerCache(torch.nn.Module, AttentionLayerBase): def forward(self): ... - def get_attn_backend(self) -> AttentionBackend: + def get_attn_backend(self) -> type[AttentionBackend]: return DeepseekV32IndexerBackend @@ -681,6 +681,7 @@ class Indexer(nn.Module): # NOTE: (zyongye) we use fp8 naive cache, # where we store value in fp8 and scale in fp32 # per self.quant_block_size element + assert cache_config is not None self.k_cache = DeepseekV32IndexerCache( head_dim=self.head_dim + self.head_dim // self.quant_block_size * 4, dtype=torch.uint8, @@ -1107,6 +1108,10 @@ class DeepseekV2MLAAttention(nn.Module): ) if self.is_v32 and (not _skip_topk or is_mtp_layer): + assert q_lora_rank is not None + assert cache_config is not None + self.indexer_rope_emb: nn.Module | None + self.indexer: Indexer | None self.indexer_rope_emb = get_rope( qk_rope_head_dim, max_position=max_position_embeddings, @@ -1218,6 +1223,7 @@ class DeepseekV2DecoderLayer(nn.Module): self.use_mha = use_mha + attn_cls: typing.Any if use_mha: attn_cls = DeepseekAttention elif model_config.use_mla: @@ -1508,7 +1514,7 @@ class DeepseekV2Model(nn.Module): rocm_aiter_moe_shared_expert_enabled = ( rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() ) - stacked_params_mapping = [ + stacked_params_mapping: list[tuple[str, str, int | str]] = [ # (param_name, shard_name, shard_id) ("gate_up_proj", "gate_proj", 0), ("gate_up_proj", "up_proj", 1), @@ -1677,7 +1683,7 @@ class DeepseekV2Model(nn.Module): # param and delegate to its expert-aware weight_loader # with expert_id. for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping + param_name, weight_name, expert_id, expert_shard_id = mapping if weight_name not in chunk_name: continue @@ -1703,7 +1709,7 @@ class DeepseekV2Model(nn.Module): param, weight_to_load, name_mapped, - shard_id=shard_id, + shard_id=expert_shard_id, expert_id=expert_id, return_success=True, ) @@ -1725,9 +1731,10 @@ class DeepseekV2Model(nn.Module): continue # Remapping the name of FP8 kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: + remapped_name = maybe_remap_kv_scale_name(name, params_dict) + if remapped_name is None: continue + name = remapped_name if is_pp_missing_parameter(name, self): continue diff --git a/vllm/model_executor/models/deepseek_vl2.py b/vllm/model_executor/models/deepseek_vl2.py index b5ad00914b..a514bef576 100644 --- a/vllm/model_executor/models/deepseek_vl2.py +++ b/vllm/model_executor/models/deepseek_vl2.py @@ -43,6 +43,7 @@ from vllm.multimodal.processing.processor import ( ) from vllm.sequence import IntermediateTensors from vllm.tokenizers import cached_tokenizer_from_config +from vllm.tokenizers.hf import HfTokenizer from vllm.transformers_utils.configs.deepseek_vl2 import ( DeepseekVLV2Config, MlpProjectorConfig, @@ -244,6 +245,7 @@ class DeepseekVL2MultiModalProcessor( ) -> BatchFeature: if not mm_data: tokenizer = self.info.get_tokenizer() + assert isinstance(tokenizer, HfTokenizer) return tokenizer(prompt, add_special_tokens=True, return_tensors="pt") processed_outputs = super()._call_hf_processor( @@ -291,6 +293,7 @@ class DeepseekVL2MultiModalProcessor( if isinstance(images, ImageEmbeddingItems): num_image_tokens = images.get_feature_size(item_idx) else: + assert isinstance(images, ImageProcessorItems) image_size = images.get_image_size(item_idx) num_image_tokens = self.info.get_num_image_tokens( @@ -406,7 +409,9 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP): return parent, names[-1] # patch for timm ViT instance to support tensor parallel - def patch_vit_for_tp(self, vit: torch.nn.Module, quant_config: QuantizationConfig): + def patch_vit_for_tp( + self, vit: torch.nn.Module, quant_config: QuantizationConfig | None + ): try: import timm except ImportError as e: diff --git a/vllm/model_executor/models/diffusion_gemma.py b/vllm/model_executor/models/diffusion_gemma.py index 259d20f99a..6395e4fddf 100644 --- a/vllm/model_executor/models/diffusion_gemma.py +++ b/vllm/model_executor/models/diffusion_gemma.py @@ -32,6 +32,7 @@ from vllm.distributed.parallel_state import get_tp_group from vllm.logger import init_logger from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, ) @@ -195,8 +196,10 @@ class DiffusionGemmaForConditionalGeneration( # ---- Vision tower ---- vision_config = getattr(config, "vision_config", None) + self.embed_vision: Gemma4MultimodalEmbedder | None if vision_config is not None: quant_config = vllm_config.quant_config + tower_quant: QuantizationConfig | None if quant_config and quant_config.get_name() in [ "bitsandbytes", "torchao", @@ -1048,7 +1051,7 @@ class DiffusionSampler: sampler: Any, diffusion_config: Any, vocab_size: int, - diffusion_states: DiffusionGemmaRequestStates | None = None, + diffusion_states: DiffusionGemmaRequestStates, *, confidence_threshold: float, t_min: float, diff --git a/vllm/model_executor/models/dots_ocr.py b/vllm/model_executor/models/dots_ocr.py index 65298a5fd3..c7bea3b780 100644 --- a/vllm/model_executor/models/dots_ocr.py +++ b/vllm/model_executor/models/dots_ocr.py @@ -149,7 +149,7 @@ class DotsOCRProcessingInfo(Qwen2VLProcessingInfo): self, **kwargs: object, ) -> Qwen2VLProcessor: - self.get_tokenizer().image_token = IMAGE_TOKEN # Ensure image token is set + setattr(self.get_tokenizer(), "image_token", IMAGE_TOKEN) # noqa: B010 processor = self.ctx.get_hf_processor( Qwen2VLProcessor, **kwargs, @@ -430,7 +430,7 @@ class DotsVisionBlock(nn.Module): *, cu_seqlens: torch.Tensor, rotary_pos_emb: torch.Tensor, - max_seqlen: int | None = None, + max_seqlen: torch.Tensor | None = None, ) -> torch.Tensor: hidden_states = hidden_states + self.attn( self.norm1(hidden_states), @@ -483,6 +483,7 @@ class DotsVisionTransformer(nn.Module): ) if require_post_norm is None: require_post_norm = len(self.blocks) == config.num_hidden_layers + self.post_trunk_norm: RMSNorm | None if require_post_norm and self.config.post_norm: self.post_trunk_norm = RMSNorm(config.embed_dim, eps=config.rms_norm_eps) else: @@ -536,7 +537,7 @@ class DotsVisionTransformer(nn.Module): rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) return rotary_pos_emb - def compute_attn_mask_seqlen(self, cu_seqlens: torch.Tensor) -> int | None: + def compute_attn_mask_seqlen(self, cu_seqlens: torch.Tensor) -> torch.Tensor | None: max_seqlen = None if self.attn_backend in { AttentionBackendEnum.FLASH_ATTN, @@ -552,15 +553,17 @@ class DotsVisionTransformer(nn.Module): rotary_pos_emb = self.rot_pos_emb(grid_thw) # Convert grid_thw to tensor (always expecting list format now) - grid_thw = torch.tensor(grid_thw, device=hidden_states.device, dtype=torch.long) + grid_thw_tensor = torch.tensor( + grid_thw, device=hidden_states.device, dtype=torch.long + ) hidden_states = hidden_states.to(self.dtype) - hidden_states = self.patch_embed(hidden_states, grid_thw) + hidden_states = self.patch_embed(hidden_states, grid_thw_tensor) cu_seqlens = torch.repeat_interleave( - grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0] + grid_thw_tensor[:, 1] * grid_thw_tensor[:, 2], grid_thw_tensor[:, 0] ).cumsum( dim=0, - dtype=grid_thw.dtype if torch.jit.is_tracing() else torch.int32, + dtype=grid_thw_tensor.dtype if torch.jit.is_tracing() else torch.int32, ) cu_seqlens = torch.cat([cu_seqlens.new_zeros(1), cu_seqlens]) @@ -620,6 +623,7 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA def get_placeholder_str(cls, modality: str, i: int) -> str | None: if modality.startswith("image"): return "<|img|><|imgpad|><|endofimg|>" + return None def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -627,6 +631,7 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA self.config: DotsOCRConfig = vllm_config.model_config.hf_config self.quant_config = vllm_config.quant_config multimodal_config = vllm_config.model_config.multimodal_config + assert multimodal_config is not None self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data" if isinstance(self.config.vision_config, dict): vision_config = DotsVisionConfig(**self.config.vision_config) @@ -677,6 +682,8 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA image_grid_thw=image_grid_thw, ) + raise AssertionError + def _process_image_input( self, image_input: DotsOCRImageInputs ) -> tuple[torch.Tensor, ...]: diff --git a/vllm/multimodal/processing/dummy_inputs.py b/vllm/multimodal/processing/dummy_inputs.py index 970cc431e5..6d3943cd91 100644 --- a/vllm/multimodal/processing/dummy_inputs.py +++ b/vllm/multimodal/processing/dummy_inputs.py @@ -118,8 +118,9 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]): width: int, height: int, num_images: int, - overrides: ImageDummyOptions | None = None, + overrides: BaseDummyOptions | None = None, ) -> list[Image.Image]: + assert overrides is None or isinstance(overrides, ImageDummyOptions) if num_images == 0: return [] if overrides: