diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index f9d064a499..7262c1f2a9 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -113,7 +113,6 @@ SEPARATE_GROUPS = [ EXCLUDE = [ r"vllm/model_executor/models/[kK]", - r"vllm/model_executor/models/[uU]", r"vllm/model_executor/models/[vV]", r"vllm/model_executor/models/[wW]", r"vllm/model_executor/models/[zZ]", diff --git a/vllm/model_executor/model_loader/reload/torchao_decorator.py b/vllm/model_executor/model_loader/reload/torchao_decorator.py index dc8e0acc6e..10ced70d31 100644 --- a/vllm/model_executor/model_loader/reload/torchao_decorator.py +++ b/vllm/model_executor/model_loader/reload/torchao_decorator.py @@ -1,10 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from collections.abc import Iterable +from collections.abc import Callable from functools import wraps -from types import FunctionType -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Concatenate, ParamSpec, TypeVar import torch @@ -20,13 +19,18 @@ if TYPE_CHECKING: __all__ = ["set_torchao_reload_attrs", "support_quantized_model_reload_from_hp_weights"] +_P = ParamSpec("_P") +_R = TypeVar("_R") + def set_torchao_reload_attrs(model: torch.nn.Module, model_config: ModelConfig): model._do_torchao_reload = True model._model_config = model_config -def support_quantized_model_reload_from_hp_weights(original_load_weights: FunctionType): +def support_quantized_model_reload_from_hp_weights( + original_load_weights: Callable[Concatenate["AutoWeightsLoader", _P], _R], +) -> Callable[Concatenate["AutoWeightsLoader", _P], _R]: """Decorator for `load_weights` method for AutoWeightsLoader.load_weights to support reloading high precision (bfloat16/float16/float32) weight for an already quantized model, this involves restoring the weights to a high precision weights and @@ -39,17 +43,16 @@ def support_quantized_model_reload_from_hp_weights(original_load_weights: Functi @wraps(original_load_weights) def patched_model_load_weights( self: "AutoWeightsLoader", - weights: Iterable[tuple[str, torch.Tensor]], - *args, - **kwargs, - ): + *args: _P.args, + **kwargs: _P.kwargs, + ) -> _R: model = self.module if not getattr(model, "_do_torchao_reload", False): - return original_load_weights(self, weights, *args, **kwargs) + return original_load_weights(self, *args, **kwargs) initialize_layerwise_reload(model) - loaded_weights = original_load_weights(self, weights, *args, **kwargs) + loaded_weights = original_load_weights(self, *args, **kwargs) finalize_layerwise_reload(model, model._model_config) return loaded_weights diff --git a/vllm/model_executor/models/phi4mm.py b/vllm/model_executor/models/phi4mm.py index 8524455975..7f223b1ccc 100644 --- a/vllm/model_executor/models/phi4mm.py +++ b/vllm/model_executor/models/phi4mm.py @@ -1297,7 +1297,7 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal): logits = self.logits_processor(self.lm_head, hidden_states) return logits - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> None: + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader(self) return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) diff --git a/vllm/model_executor/models/ultravox.py b/vllm/model_executor/models/ultravox.py index a454ba5962..72ce26f21c 100644 --- a/vllm/model_executor/models/ultravox.py +++ b/vllm/model_executor/models/ultravox.py @@ -282,10 +282,14 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo def get_replacement_ultravox(item_idx: int): if "audio_num_tokens" in out_mm_data: - return [replacement_id] * int(out_mm_data["audio_num_tokens"][item_idx]) + audio_num_tokens = out_mm_data["audio_num_tokens"] + assert isinstance(audio_num_tokens, torch.Tensor) + return [replacement_id] * int(audio_num_tokens[item_idx]) start = chunks_start_idx[item_idx] end = chunks_start_idx[item_idx + 1] - audio_token_len = out_mm_data["audio_token_len"][start:end].sum() + audio_token_lens = out_mm_data["audio_token_len"] + assert isinstance(audio_token_lens, torch.Tensor) + audio_token_len = audio_token_lens[start:end].sum() return [replacement_id] * int(audio_token_len) # type: ignore return [ @@ -566,7 +570,8 @@ class UltravoxWhisperEncoder(WhisperEncoder): ) -> torch.Tensor: return (input_lengths - 1) // 2 + 1 - def forward( + # Ultravox requires chunk lengths in addition to Whisper's input features. + def forward( # type: ignore[override] self, input_features: torch.Tensor, audio_lens: torch.Tensor, @@ -960,12 +965,12 @@ def pad_and_concat_to_dim3( return features - features = [pad_and_concat_to_dim3(f) for f in features] + tensors = [pad_and_concat_to_dim3(f) for f in features] - max_len = max(f.shape[-1] for f in features) + max_len = max(f.shape[-1] for f in tensors) # Ensure all features have dim=3 - features = [f.view(-1, *f.shape[-2:]) for f in features] + tensors = [f.view(-1, *f.shape[-2:]) for f in tensors] # Pad and concatenate: # [[B1, 80, M1], [B2, 80, M2]] -> [B1+B2, 80, max(M1, M2)] - features = [F.pad(f, (0, max_len - f.shape[-1])) for f in features] - return torch.cat(features) + tensors = [F.pad(f, (0, max_len - f.shape[-1])) for f in tensors] + return torch.cat(tensors) diff --git a/vllm/model_executor/models/unlimited_ocr.py b/vllm/model_executor/models/unlimited_ocr.py index d248e5c8e0..27e5503d2c 100644 --- a/vllm/model_executor/models/unlimited_ocr.py +++ b/vllm/model_executor/models/unlimited_ocr.py @@ -199,6 +199,7 @@ class UnlimitedOCRMultiModalProcessor(DeepseekOCRMultiModalProcessor): if isinstance(images, ImageEmbeddingItems): num_image_tokens = images.get_feature_size(item_idx) else: + assert isinstance(images, ImageProcessorItems) size = images.get_image_size(item_idx) # Disable crop mode for multi-image input. diff --git a/vllm/model_executor/models/utils.py b/vllm/model_executor/models/utils.py index 6375e0572e..e762ad1c47 100644 --- a/vllm/model_executor/models/utils.py +++ b/vllm/model_executor/models/utils.py @@ -35,7 +35,6 @@ if TYPE_CHECKING: from transformers import PreTrainedConfig from transformers.conversion_mapping import WeightRenaming - from vllm.config.model import ModelConfig from vllm.model_executor.layers.quantization import QuantizationConfig logger = init_logger(__name__) @@ -337,7 +336,7 @@ class AutoWeightsLoader: """ # Add persistent registered buffers. # Non-persistent buffers are excluded, matching PyTorch state_dict(). - non_persistent = getattr(module, "_non_persistent_buffers_set", set()) + non_persistent: set[str] = getattr(module, "_non_persistent_buffers_set", set()) for buf_name, buf in module.named_buffers(recurse=False): if buf_name not in child_params and buf_name not in non_persistent: child_params[buf_name] = buf @@ -558,7 +557,7 @@ def maybe_fuse_shared_experts( def get_spec_layer_idx_from_weight_name( - config: "ModelConfig", weight_name: str + config: "PreTrainedConfig", weight_name: str ) -> int | None: """Return the MTP layer index a weight belongs to, or None. @@ -587,7 +586,7 @@ def get_spec_layer_idx_from_weight_name( def skip_spec_layers( - weights: Iterable[tuple[str, torch.Tensor]], config: "ModelConfig" + weights: Iterable[tuple[str, torch.Tensor]], config: "PreTrainedConfig" ) -> Iterable[tuple[str, torch.Tensor]]: """Drop MTP spec-layer weights (loaded by the MTP head, not the base model). @@ -629,11 +628,11 @@ def init_vllm_registered_model( @overload -def flatten_bn(x: torch.Tensor) -> torch.Tensor: ... +def flatten_bn(x: list[torch.Tensor]) -> list[torch.Tensor]: ... @overload -def flatten_bn(x: list[torch.Tensor]) -> list[torch.Tensor]: ... +def flatten_bn(x: torch.Tensor) -> torch.Tensor: ... @overload @@ -973,7 +972,9 @@ def get_draft_quant_config(vllm_config: VllmConfig) -> "QuantizationConfig | Non The draft model's config if available, None otherwise. """ - draft_model_config = vllm_config.speculative_config.draft_model_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + draft_model_config = speculative_config.draft_model_config draft_load_config = vllm_config.load_config return (