diff --git a/tests/models/multimodal/generation/test_voxtral_realtime.py b/tests/models/multimodal/generation/test_voxtral_realtime.py index 2c13e07ff3..09432b4494 100644 --- a/tests/models/multimodal/generation/test_voxtral_realtime.py +++ b/tests/models/multimodal/generation/test_voxtral_realtime.py @@ -16,7 +16,9 @@ from vllm import LLM, SamplingParams from vllm.assets.audio import AudioAsset from vllm.config import CUDAGraphMode from vllm.engine.arg_utils import AsyncEngineArgs +from vllm.engine.protocol import StreamingInput from vllm.platforms import current_platform +from vllm.renderers.inputs.preprocess import parse_model_prompt from vllm.utils.math_utils import cdiv from vllm.v1.engine.async_llm import AsyncLLM from vllm.v1.kv_cache_interface import SlidingWindowSpec @@ -222,6 +224,14 @@ async def test_voxtral_realtime_generator(audio_assets, tokenizer, async_engine) sampling_params = SamplingParams(temperature=0.0, max_tokens=1) audio_config = tokenizer.instruct_tokenizer.audio_encoder.audio_config + async def input_stream(audio_buffer): + async for prompt in audio_buffer.get_input_stream(): + parsed_prompt = parse_model_prompt(async_engine.model_config, prompt) + (engine_input,) = await async_engine.renderer.render_cmpl_async( + [parsed_prompt] + ) + yield StreamingInput(prompt=engine_input) + output_tokens_list = [] for i, audio_asset in enumerate(audio_assets): output_tokens = [] @@ -241,7 +251,7 @@ async def test_voxtral_realtime_generator(audio_assets, tokenizer, async_engine) request_id = f"session-{i}" async for resp in async_engine.generate( - prompt=buffer.get_input_stream(), + prompt=input_stream(buffer), sampling_params=sampling_params, request_id=request_id, ): diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index 7262c1f2a9..69b36d43b2 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -113,7 +113,6 @@ SEPARATE_GROUPS = [ EXCLUDE = [ r"vllm/model_executor/models/[kK]", - r"vllm/model_executor/models/[vV]", r"vllm/model_executor/models/[wW]", r"vllm/model_executor/models/[zZ]", ] diff --git a/vllm/model_executor/models/vision.py b/vllm/model_executor/models/vision.py index b35db4de27..d16e631872 100644 --- a/vllm/model_executor/models/vision.py +++ b/vllm/model_executor/models/vision.py @@ -5,7 +5,7 @@ import itertools import math from abc import ABC, abstractmethod from collections.abc import Callable -from typing import Final, Generic, Literal, Protocol, TypeAlias, TypeVar +from typing import Generic, Literal, Protocol, TypeAlias, TypeVar import torch from transformers import PreTrainedConfig @@ -24,10 +24,12 @@ from vllm.v1.attention.backends.registry import AttentionBackendEnum logger = init_logger(__name__) _C = TypeVar("_C", bound=PreTrainedConfig) +_C_co = TypeVar("_C_co", bound=PreTrainedConfig, covariant=True) -class _RootConfig(Protocol[_C]): - vision_config: _C +class _RootConfig(Protocol[_C_co]): + @property + def vision_config(self) -> _C_co: ... class VisionEncoderInfo(ABC, Generic[_C]): @@ -60,7 +62,8 @@ class VisionEncoderInfo(ABC, Generic[_C]): class VisionLanguageConfig(Protocol): - vision_config: Final[PreTrainedConfig] + @property + def vision_config(self) -> PreTrainedConfig: ... def get_vision_encoder_info(hf_config: VisionLanguageConfig) -> VisionEncoderInfo: diff --git a/vllm/model_executor/models/voxtral.py b/vllm/model_executor/models/voxtral.py index 21b291344d..c65dc38298 100644 --- a/vllm/model_executor/models/voxtral.py +++ b/vllm/model_executor/models/voxtral.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import math -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Callable, Iterable, Mapping, Sequence from functools import partial from typing import cast @@ -317,7 +317,7 @@ class VoxtralForConditionalGeneration( # update quant config to so that ignored module and target module names # match the vLLM model names - if hasattr(vllm_config, "quant_config"): + if vllm_config.quant_config is not None: vllm_config.quant_config = self.maybe_update_quant_config( vllm_config.quant_config ) @@ -724,6 +724,7 @@ class VoxtralEncoderModel(nn.Module): self.config = cast(WhisperConfig, vllm_config.model_config.hf_config) self.dtype: torch.dtype = vllm_config.model_config.dtype self.is_causal = getattr(self.config, "is_causal", False) + WhisperEncoderCls: Callable[..., WhisperEncoder | WhisperCausalEncoder] if self.is_causal: WhisperEncoderCls = WhisperCausalEncoder else: @@ -830,7 +831,7 @@ class VoxtralEncoderModel(nn.Module): return results def load_weight(self, weight: tuple[str, torch.Tensor]) -> str: - stacked_params_mapping = [ + stacked_params_mapping: list[tuple[str, str, str | int]] = [ # (param_name, shard_name, shard_id) ("qkv_proj", "q_proj", "q"), ("qkv_proj", "k_proj", "k"), diff --git a/vllm/model_executor/models/voxtral_realtime.py b/vllm/model_executor/models/voxtral_realtime.py index 9fdc946df9..8531d7d18f 100644 --- a/vllm/model_executor/models/voxtral_realtime.py +++ b/vllm/model_executor/models/voxtral_realtime.py @@ -16,7 +16,6 @@ from mistral_common.tokens.tokenizers.audio import Audio, AudioConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig from vllm.config.speech_to_text import SpeechToTextParams -from vllm.engine.protocol import StreamingInput from vllm.inputs import PromptType, TokensPrompt from vllm.logger import init_logger from vllm.model_executor.models.interfaces import MultiModalEmbeddings, SupportsRealtime @@ -26,6 +25,7 @@ from vllm.model_executor.models.voxtral import ( VoxtralMultiModalProcessor, VoxtralProcessingInfo, ) +from vllm.model_executor.models.whisper_causal import WhisperCausalEncoder from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.parse import MultiModalDataItems from vllm.multimodal.processing import ProcessorInputs, TimingContext @@ -56,7 +56,7 @@ class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor): mm_items: MultiModalDataItems, mm_res: MultiModalProcessingResult, ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: - mm_kwargs = mm_res.kwargs + mm_kwargs = mm_res.kwargs.require_data() # there are no placeholder audio tokens for streaming # so we need to build the place placeholder positions manually @@ -69,7 +69,9 @@ class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor): tokenizer = self.info.get_tokenizer() audio_config = tokenizer.instruct.audio_encoder.audio_config - num_audio_samples = audios[0]["audio_arrays"].data.shape[0] + audio_array = audios[0]["audio_arrays"].data + assert isinstance(audio_array, (torch.Tensor, np.ndarray)) + num_audio_samples = audio_array.shape[0] length = audio_config.num_audio_tokens(num_audio_samples) features_info = PlaceholderFeaturesInfo( @@ -162,7 +164,7 @@ class VoxtralRealtimeBuffer: for token in tokens: await self._token_queue.put(token) - async def get_input_stream(self) -> AsyncGenerator[StreamingInput]: + async def get_input_stream(self) -> AsyncGenerator[TokensPrompt]: for frame_size, num_tokens in self._generate_frame_size_and_num_tokens(): next_tokens = [await self._token_queue.get() for _ in range(num_tokens)] @@ -188,11 +190,9 @@ class VoxtralRealtimeBuffer: self._leftover = audio_array[stride:] - yield StreamingInput( - TokensPrompt( - prompt_token_ids=next_tokens, - multi_modal_data={"audio": (frame, None)}, - ) + yield TokensPrompt( + prompt_token_ids=next_tokens, + multi_modal_data={"audio": (frame, None)}, ) @@ -274,8 +274,8 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim token_task = asyncio.create_task(feed_tokens()) try: - async for streaming_input in buffer.get_input_stream(): - yield streaming_input.prompt + async for prompt in buffer.get_input_stream(): + yield prompt finally: audio_task.cancel() token_task.cancel() @@ -420,6 +420,7 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim seq_lens = [mel.shape[1] for mel in mel_features] # [total_num_20ms_frames, hidden_size] + assert isinstance(self.whisper_encoder.whisper_encoder, WhisperCausalEncoder) audio_embeddings = self.whisper_encoder.whisper_encoder.forward_conv( mel_features )