[Mypy] Fix mypy typing for Voxtral and vision models (#58251)

Signed-off-by: Ashraf Bhuiyan <mbhuiyan@redhat.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
Ashraf Bhuiyan
2026-09-28 15:52:50 -04:00
committed by GitHub
co-authored by Wentao Ye
parent 2840ca7a33
commit 2619b6b72b
5 changed files with 34 additions and 20 deletions
@@ -16,7 +16,9 @@ from vllm import LLM, SamplingParams
from vllm.assets.audio import AudioAsset
from vllm.config import CUDAGraphMode
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.engine.protocol import StreamingInput
from vllm.platforms import current_platform
from vllm.renderers.inputs.preprocess import parse_model_prompt
from vllm.utils.math_utils import cdiv
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.kv_cache_interface import SlidingWindowSpec
@@ -222,6 +224,14 @@ async def test_voxtral_realtime_generator(audio_assets, tokenizer, async_engine)
sampling_params = SamplingParams(temperature=0.0, max_tokens=1)
audio_config = tokenizer.instruct_tokenizer.audio_encoder.audio_config
async def input_stream(audio_buffer):
async for prompt in audio_buffer.get_input_stream():
parsed_prompt = parse_model_prompt(async_engine.model_config, prompt)
(engine_input,) = await async_engine.renderer.render_cmpl_async(
[parsed_prompt]
)
yield StreamingInput(prompt=engine_input)
output_tokens_list = []
for i, audio_asset in enumerate(audio_assets):
output_tokens = []
@@ -241,7 +251,7 @@ async def test_voxtral_realtime_generator(audio_assets, tokenizer, async_engine)
request_id = f"session-{i}"
async for resp in async_engine.generate(
prompt=buffer.get_input_stream(),
prompt=input_stream(buffer),
sampling_params=sampling_params,
request_id=request_id,
):
-1
View File
@@ -113,7 +113,6 @@ SEPARATE_GROUPS = [
EXCLUDE = [
r"vllm/model_executor/models/[kK]",
r"vllm/model_executor/models/[vV]",
r"vllm/model_executor/models/[wW]",
r"vllm/model_executor/models/[zZ]",
]
+7 -4
View File
@@ -5,7 +5,7 @@ import itertools
import math
from abc import ABC, abstractmethod
from collections.abc import Callable
from typing import Final, Generic, Literal, Protocol, TypeAlias, TypeVar
from typing import Generic, Literal, Protocol, TypeAlias, TypeVar
import torch
from transformers import PreTrainedConfig
@@ -24,10 +24,12 @@ from vllm.v1.attention.backends.registry import AttentionBackendEnum
logger = init_logger(__name__)
_C = TypeVar("_C", bound=PreTrainedConfig)
_C_co = TypeVar("_C_co", bound=PreTrainedConfig, covariant=True)
class _RootConfig(Protocol[_C]):
vision_config: _C
class _RootConfig(Protocol[_C_co]):
@property
def vision_config(self) -> _C_co: ...
class VisionEncoderInfo(ABC, Generic[_C]):
@@ -60,7 +62,8 @@ class VisionEncoderInfo(ABC, Generic[_C]):
class VisionLanguageConfig(Protocol):
vision_config: Final[PreTrainedConfig]
@property
def vision_config(self) -> PreTrainedConfig: ...
def get_vision_encoder_info(hf_config: VisionLanguageConfig) -> VisionEncoderInfo:
+4 -3
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from collections.abc import Iterable, Mapping, Sequence
from collections.abc import Callable, Iterable, Mapping, Sequence
from functools import partial
from typing import cast
@@ -317,7 +317,7 @@ class VoxtralForConditionalGeneration(
# update quant config to so that ignored module and target module names
# match the vLLM model names
if hasattr(vllm_config, "quant_config"):
if vllm_config.quant_config is not None:
vllm_config.quant_config = self.maybe_update_quant_config(
vllm_config.quant_config
)
@@ -724,6 +724,7 @@ class VoxtralEncoderModel(nn.Module):
self.config = cast(WhisperConfig, vllm_config.model_config.hf_config)
self.dtype: torch.dtype = vllm_config.model_config.dtype
self.is_causal = getattr(self.config, "is_causal", False)
WhisperEncoderCls: Callable[..., WhisperEncoder | WhisperCausalEncoder]
if self.is_causal:
WhisperEncoderCls = WhisperCausalEncoder
else:
@@ -830,7 +831,7 @@ class VoxtralEncoderModel(nn.Module):
return results
def load_weight(self, weight: tuple[str, torch.Tensor]) -> str:
stacked_params_mapping = [
stacked_params_mapping: list[tuple[str, str, str | int]] = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
+12 -11
View File
@@ -16,7 +16,6 @@ from mistral_common.tokens.tokenizers.audio import Audio, AudioConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.engine.protocol import StreamingInput
from vllm.inputs import PromptType, TokensPrompt
from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import MultiModalEmbeddings, SupportsRealtime
@@ -26,6 +25,7 @@ from vllm.model_executor.models.voxtral import (
VoxtralMultiModalProcessor,
VoxtralProcessingInfo,
)
from vllm.model_executor.models.whisper_causal import WhisperCausalEncoder
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.parse import MultiModalDataItems
from vllm.multimodal.processing import ProcessorInputs, TimingContext
@@ -56,7 +56,7 @@ class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor):
mm_items: MultiModalDataItems,
mm_res: MultiModalProcessingResult,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
mm_kwargs = mm_res.kwargs
mm_kwargs = mm_res.kwargs.require_data()
# there are no placeholder audio tokens for streaming
# so we need to build the place placeholder positions manually
@@ -69,7 +69,9 @@ class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor):
tokenizer = self.info.get_tokenizer()
audio_config = tokenizer.instruct.audio_encoder.audio_config
num_audio_samples = audios[0]["audio_arrays"].data.shape[0]
audio_array = audios[0]["audio_arrays"].data
assert isinstance(audio_array, (torch.Tensor, np.ndarray))
num_audio_samples = audio_array.shape[0]
length = audio_config.num_audio_tokens(num_audio_samples)
features_info = PlaceholderFeaturesInfo(
@@ -162,7 +164,7 @@ class VoxtralRealtimeBuffer:
for token in tokens:
await self._token_queue.put(token)
async def get_input_stream(self) -> AsyncGenerator[StreamingInput]:
async def get_input_stream(self) -> AsyncGenerator[TokensPrompt]:
for frame_size, num_tokens in self._generate_frame_size_and_num_tokens():
next_tokens = [await self._token_queue.get() for _ in range(num_tokens)]
@@ -188,11 +190,9 @@ class VoxtralRealtimeBuffer:
self._leftover = audio_array[stride:]
yield StreamingInput(
TokensPrompt(
prompt_token_ids=next_tokens,
multi_modal_data={"audio": (frame, None)},
)
yield TokensPrompt(
prompt_token_ids=next_tokens,
multi_modal_data={"audio": (frame, None)},
)
@@ -274,8 +274,8 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim
token_task = asyncio.create_task(feed_tokens())
try:
async for streaming_input in buffer.get_input_stream():
yield streaming_input.prompt
async for prompt in buffer.get_input_stream():
yield prompt
finally:
audio_task.cancel()
token_task.cancel()
@@ -420,6 +420,7 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim
seq_lens = [mel.shape[1] for mel in mel_features]
# [total_num_20ms_frames, hidden_size]
assert isinstance(self.whisper_encoder.whisper_encoder, WhisperCausalEncoder)
audio_embeddings = self.whisper_encoder.whisper_encoder.forward_conv(
mel_features
)