[Mypy] Fix mypy typing for Voxtral and vision models (#58251)
Signed-off-by: Ashraf Bhuiyan <mbhuiyan@redhat.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
Wentao Ye
parent
2840ca7a33
commit
2619b6b72b
@@ -16,7 +16,9 @@ from vllm import LLM, SamplingParams
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.engine.protocol import StreamingInput
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.renderers.inputs.preprocess import parse_model_prompt
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.kv_cache_interface import SlidingWindowSpec
|
||||
@@ -222,6 +224,14 @@ async def test_voxtral_realtime_generator(audio_assets, tokenizer, async_engine)
|
||||
sampling_params = SamplingParams(temperature=0.0, max_tokens=1)
|
||||
audio_config = tokenizer.instruct_tokenizer.audio_encoder.audio_config
|
||||
|
||||
async def input_stream(audio_buffer):
|
||||
async for prompt in audio_buffer.get_input_stream():
|
||||
parsed_prompt = parse_model_prompt(async_engine.model_config, prompt)
|
||||
(engine_input,) = await async_engine.renderer.render_cmpl_async(
|
||||
[parsed_prompt]
|
||||
)
|
||||
yield StreamingInput(prompt=engine_input)
|
||||
|
||||
output_tokens_list = []
|
||||
for i, audio_asset in enumerate(audio_assets):
|
||||
output_tokens = []
|
||||
@@ -241,7 +251,7 @@ async def test_voxtral_realtime_generator(audio_assets, tokenizer, async_engine)
|
||||
request_id = f"session-{i}"
|
||||
|
||||
async for resp in async_engine.generate(
|
||||
prompt=buffer.get_input_stream(),
|
||||
prompt=input_stream(buffer),
|
||||
sampling_params=sampling_params,
|
||||
request_id=request_id,
|
||||
):
|
||||
|
||||
@@ -113,7 +113,6 @@ SEPARATE_GROUPS = [
|
||||
|
||||
EXCLUDE = [
|
||||
r"vllm/model_executor/models/[kK]",
|
||||
r"vllm/model_executor/models/[vV]",
|
||||
r"vllm/model_executor/models/[wW]",
|
||||
r"vllm/model_executor/models/[zZ]",
|
||||
]
|
||||
|
||||
@@ -5,7 +5,7 @@ import itertools
|
||||
import math
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable
|
||||
from typing import Final, Generic, Literal, Protocol, TypeAlias, TypeVar
|
||||
from typing import Generic, Literal, Protocol, TypeAlias, TypeVar
|
||||
|
||||
import torch
|
||||
from transformers import PreTrainedConfig
|
||||
@@ -24,10 +24,12 @@ from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_C = TypeVar("_C", bound=PreTrainedConfig)
|
||||
_C_co = TypeVar("_C_co", bound=PreTrainedConfig, covariant=True)
|
||||
|
||||
|
||||
class _RootConfig(Protocol[_C]):
|
||||
vision_config: _C
|
||||
class _RootConfig(Protocol[_C_co]):
|
||||
@property
|
||||
def vision_config(self) -> _C_co: ...
|
||||
|
||||
|
||||
class VisionEncoderInfo(ABC, Generic[_C]):
|
||||
@@ -60,7 +62,8 @@ class VisionEncoderInfo(ABC, Generic[_C]):
|
||||
|
||||
|
||||
class VisionLanguageConfig(Protocol):
|
||||
vision_config: Final[PreTrainedConfig]
|
||||
@property
|
||||
def vision_config(self) -> PreTrainedConfig: ...
|
||||
|
||||
|
||||
def get_vision_encoder_info(hf_config: VisionLanguageConfig) -> VisionEncoderInfo:
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
||||
from functools import partial
|
||||
from typing import cast
|
||||
|
||||
@@ -317,7 +317,7 @@ class VoxtralForConditionalGeneration(
|
||||
|
||||
# update quant config to so that ignored module and target module names
|
||||
# match the vLLM model names
|
||||
if hasattr(vllm_config, "quant_config"):
|
||||
if vllm_config.quant_config is not None:
|
||||
vllm_config.quant_config = self.maybe_update_quant_config(
|
||||
vllm_config.quant_config
|
||||
)
|
||||
@@ -724,6 +724,7 @@ class VoxtralEncoderModel(nn.Module):
|
||||
self.config = cast(WhisperConfig, vllm_config.model_config.hf_config)
|
||||
self.dtype: torch.dtype = vllm_config.model_config.dtype
|
||||
self.is_causal = getattr(self.config, "is_causal", False)
|
||||
WhisperEncoderCls: Callable[..., WhisperEncoder | WhisperCausalEncoder]
|
||||
if self.is_causal:
|
||||
WhisperEncoderCls = WhisperCausalEncoder
|
||||
else:
|
||||
@@ -830,7 +831,7 @@ class VoxtralEncoderModel(nn.Module):
|
||||
return results
|
||||
|
||||
def load_weight(self, weight: tuple[str, torch.Tensor]) -> str:
|
||||
stacked_params_mapping = [
|
||||
stacked_params_mapping: list[tuple[str, str, str | int]] = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
|
||||
@@ -16,7 +16,6 @@ from mistral_common.tokens.tokenizers.audio import Audio, AudioConfig
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
||||
from vllm.config.speech_to_text import SpeechToTextParams
|
||||
from vllm.engine.protocol import StreamingInput
|
||||
from vllm.inputs import PromptType, TokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import MultiModalEmbeddings, SupportsRealtime
|
||||
@@ -26,6 +25,7 @@ from vllm.model_executor.models.voxtral import (
|
||||
VoxtralMultiModalProcessor,
|
||||
VoxtralProcessingInfo,
|
||||
)
|
||||
from vllm.model_executor.models.whisper_causal import WhisperCausalEncoder
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.parse import MultiModalDataItems
|
||||
from vllm.multimodal.processing import ProcessorInputs, TimingContext
|
||||
@@ -56,7 +56,7 @@ class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor):
|
||||
mm_items: MultiModalDataItems,
|
||||
mm_res: MultiModalProcessingResult,
|
||||
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
||||
mm_kwargs = mm_res.kwargs
|
||||
mm_kwargs = mm_res.kwargs.require_data()
|
||||
|
||||
# there are no placeholder audio tokens for streaming
|
||||
# so we need to build the place placeholder positions manually
|
||||
@@ -69,7 +69,9 @@ class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor):
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
audio_config = tokenizer.instruct.audio_encoder.audio_config
|
||||
|
||||
num_audio_samples = audios[0]["audio_arrays"].data.shape[0]
|
||||
audio_array = audios[0]["audio_arrays"].data
|
||||
assert isinstance(audio_array, (torch.Tensor, np.ndarray))
|
||||
num_audio_samples = audio_array.shape[0]
|
||||
length = audio_config.num_audio_tokens(num_audio_samples)
|
||||
|
||||
features_info = PlaceholderFeaturesInfo(
|
||||
@@ -162,7 +164,7 @@ class VoxtralRealtimeBuffer:
|
||||
for token in tokens:
|
||||
await self._token_queue.put(token)
|
||||
|
||||
async def get_input_stream(self) -> AsyncGenerator[StreamingInput]:
|
||||
async def get_input_stream(self) -> AsyncGenerator[TokensPrompt]:
|
||||
for frame_size, num_tokens in self._generate_frame_size_and_num_tokens():
|
||||
next_tokens = [await self._token_queue.get() for _ in range(num_tokens)]
|
||||
|
||||
@@ -188,11 +190,9 @@ class VoxtralRealtimeBuffer:
|
||||
|
||||
self._leftover = audio_array[stride:]
|
||||
|
||||
yield StreamingInput(
|
||||
TokensPrompt(
|
||||
prompt_token_ids=next_tokens,
|
||||
multi_modal_data={"audio": (frame, None)},
|
||||
)
|
||||
yield TokensPrompt(
|
||||
prompt_token_ids=next_tokens,
|
||||
multi_modal_data={"audio": (frame, None)},
|
||||
)
|
||||
|
||||
|
||||
@@ -274,8 +274,8 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim
|
||||
token_task = asyncio.create_task(feed_tokens())
|
||||
|
||||
try:
|
||||
async for streaming_input in buffer.get_input_stream():
|
||||
yield streaming_input.prompt
|
||||
async for prompt in buffer.get_input_stream():
|
||||
yield prompt
|
||||
finally:
|
||||
audio_task.cancel()
|
||||
token_task.cancel()
|
||||
@@ -420,6 +420,7 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim
|
||||
|
||||
seq_lens = [mel.shape[1] for mel in mel_features]
|
||||
# [total_num_20ms_frames, hidden_size]
|
||||
assert isinstance(self.whisper_encoder.whisper_encoder, WhisperCausalEncoder)
|
||||
audio_embeddings = self.whisper_encoder.whisper_encoder.forward_conv(
|
||||
mel_features
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user