diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index 69b36d43b2..5ad37e138e 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -113,7 +113,6 @@ SEPARATE_GROUPS = [ EXCLUDE = [ r"vllm/model_executor/models/[kK]", - r"vllm/model_executor/models/[wW]", r"vllm/model_executor/models/[zZ]", ] diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index 7901a136f5..e28cc9b806 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -61,6 +61,7 @@ if TYPE_CHECKING: from vllm.multimodal.registry import _ProcessorFactories from vllm.sequence import IntermediateTensors from vllm.tasks import ScoreType + from vllm.tokenizers import TokenizerLike from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum from vllm.v1.worker.encoder_cudagraph_defs import ( EncoderCudaGraphCaptureInputs, @@ -1449,7 +1450,7 @@ class SupportsTranscription(Protocol): def parse_language_detection_output( cls, token_ids: list[int], - tokenizer: object, + tokenizer: "TokenizerLike", ) -> str: """Parse the detected language from model output token IDs. @@ -1461,7 +1462,7 @@ class SupportsTranscription(Protocol): @classmethod def get_language_token_ids( cls, - tokenizer: object, + tokenizer: "TokenizerLike", ) -> list[int] | None: """Return token IDs that represent valid language tokens. diff --git a/vllm/model_executor/models/whisper.py b/vllm/model_executor/models/whisper.py index f67e3f0cb9..09f5f85453 100644 --- a/vllm/model_executor/models/whisper.py +++ b/vllm/model_executor/models/whisper.py @@ -63,6 +63,7 @@ from vllm.multimodal.processing import ( ) from vllm.multimodal.processing.processor import HFMultiModalInputs from vllm.renderers import TokenizeParams +from vllm.tokenizers import TokenizerLike from vllm.transformers_utils.processor import cached_processor_from_config from vllm.utils.jsontree import json_map_leaves from vllm.utils.tensor_schema import TensorSchema, TensorShape @@ -325,7 +326,7 @@ class WhisperCrossAttention(WhisperAttention): prefix=f"{prefix}.kv_proj", ) - def forward( + def forward( # type: ignore[override] self, hidden_states: torch.Tensor, encoder_hidden_states: torch.Tensor | None, @@ -872,7 +873,7 @@ class WhisperForConditionalGeneration( @classmethod def get_language_token_ids( cls, - tokenizer: object, + tokenizer: TokenizerLike, ) -> list[int]: """Return token IDs for all supported language tokens. @@ -908,8 +909,8 @@ class WhisperForConditionalGeneration( def parse_language_detection_output( cls, token_ids: list[int], - tokenizer: object, - ) -> str | None: + tokenizer: TokenizerLike, + ) -> str: """Parse the language token predicted by Whisper. Decodes the first token ID and extracts the language code from the @@ -1004,6 +1005,7 @@ class WhisperForConditionalGeneration( audio_input = self._parse_and_validate_audio_input(**kwargs) # Split concatenated encoder outputs into one tensor per audio input enc_output = self.model.get_encoder_outputs(audio_input["input_features"]) + assert enc_output is not None # The assumption is we can only process whole mm items (audios) return enc_output.unbind(dim=0) @@ -1021,8 +1023,12 @@ class WhisperForConditionalGeneration( def _parse_and_validate_audio_input(self, **kwargs: object) -> WhisperAudioInputs: input_features = kwargs.pop("input_features", None) + def to_dtype(value: object) -> torch.Tensor: + assert isinstance(value, torch.Tensor) + return value.to(self.dtype) + if input_features is not None: - input_features = json_map_leaves(lambda x: x.to(self.dtype), input_features) + input_features = json_map_leaves(to_dtype, input_features) return WhisperAudioInputs(input_features=input_features) diff --git a/vllm/model_executor/models/whisper_causal.py b/vllm/model_executor/models/whisper_causal.py index ec84cae8cf..745945c2ea 100644 --- a/vllm/model_executor/models/whisper_causal.py +++ b/vllm/model_executor/models/whisper_causal.py @@ -6,7 +6,7 @@ import math from dataclasses import replace from fractions import Fraction from functools import partial -from typing import ClassVar +from typing import ClassVar, Protocol import torch import torch.nn.functional as F @@ -35,11 +35,6 @@ from vllm.v1.attention.backend import ( ) from vllm.v1.attention.backends.cpu_attn import CPUAttentionBackend from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend - -try: - from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionBackend -except ImportError: - AiterFlashAttentionBackend = None from vllm.v1.attention.backends.rocm_attn import RocmAttentionBackend from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend from vllm.v1.attention.selector import get_attn_backend @@ -47,11 +42,24 @@ from vllm.v1.kv_cache_interface import AttentionSpec, SlidingWindowSpec from .utils import make_layers +AiterFlashAttentionBackend: type[AttentionBackend] | None = None +try: + from vllm.v1.attention.backends import rocm_aiter_fa + + AiterFlashAttentionBackend = rocm_aiter_fa.AiterFlashAttentionBackend +except ImportError: + pass + logger = logging.getLogger(__name__) CausalRMSNorm = partial(RMSNorm, eps=1e-5) +class _BlockPoolingAttentionMetadata(Protocol): + @property + def slot_mapping(self) -> torch.Tensor: ... + + def _pad1d( x: torch.Tensor, paddings: tuple[int, int], @@ -115,7 +123,7 @@ class WhisperCausalConv1d(nn.Conv1d): @functools.lru_cache def create_whisper_attention_backend_with_block_pooling( - underlying_attn_backend: AttentionBackend, + underlying_attn_backend: type[AttentionBackend], block_pool_size: int, sliding_window: int | None = None, ) -> type[AttentionBackend]: @@ -270,7 +278,7 @@ def create_whisper_attention_backend_with_block_pooling( key: torch.Tensor, value: torch.Tensor, kv_cache: torch.Tensor, - attn_metadata: AttentionMetadata, + attn_metadata: _BlockPoolingAttentionMetadata | None, output: torch.Tensor, output_scale: torch.Tensor | None = None, output_block_scale: torch.Tensor | None = None,