[Mypy Fix] Mypy fix for "vllm/model_executor/models/[eE][fF]" (#53381)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-08-22 15:41:27 -04:00
committed by GitHub
parent 236f78cc5c
commit 811d12ee26
16 changed files with 71 additions and 84 deletions
-2
View File
@@ -112,8 +112,6 @@ SEPARATE_GROUPS = [
] ]
EXCLUDE = [ EXCLUDE = [
r"vllm/model_executor/models/[eE]",
r"vllm/model_executor/models/[fF]",
r"vllm/model_executor/models/[gG]", r"vllm/model_executor/models/[gG]",
r"vllm/model_executor/models/[hH]", r"vllm/model_executor/models/[hH]",
r"vllm/model_executor/models/[iI]", r"vllm/model_executor/models/[iI]",
+4 -2
View File
@@ -106,8 +106,10 @@ def support_torch_compile(
@overload @overload
def support_torch_compile( def support_torch_compile(
*, *,
dynamic_arg_dims: dict[str, int | list[int] | dict[int, str]] | None, dynamic_arg_dims: dict[str, int | list[int] | dict[int, str]] | None = None,
mark_unbacked_dims: dict[str, int | list[int]] | None, mark_unbacked_dims: dict[str, int | list[int]] | None = None,
enable_if: Callable[[VllmConfig], bool] | None = None,
is_encoder: bool = False,
) -> Callable[[type[_T]], type[_T]]: ... ) -> Callable[[type[_T]], type[_T]]: ...
+2 -1
View File
@@ -150,6 +150,7 @@ class Eagle2_5_VLForConditionalGeneration(
config = vllm_config.model_config.hf_config config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config quant_config = vllm_config.quant_config
multimodal_config = vllm_config.model_config.multimodal_config multimodal_config = vllm_config.model_config.multimodal_config
assert multimodal_config is not None
self.config = config self.config = config
self.multimodal_config = multimodal_config self.multimodal_config = multimodal_config
@@ -187,7 +188,7 @@ class Eagle2_5_VLForConditionalGeneration(
prefix=maybe_prefix(prefix, "language_model"), prefix=maybe_prefix(prefix, "language_model"),
) )
self.img_context_token_id = None self.img_context_token_id: int | None = None
self.make_empty_intermediate_tensors = ( self.make_empty_intermediate_tensors = (
self.language_model.make_empty_intermediate_tensors self.language_model.make_empty_intermediate_tensors
+2 -1
View File
@@ -127,7 +127,7 @@ class Ernie4_5_MoeMoE(nn.Module):
self.layer_idx = layer_idx self.layer_idx = layer_idx
self.tp_size = get_tensor_model_parallel_world_size() self.tp_size = get_tensor_model_parallel_world_size()
self.moe_num_shared_experts = getattr(config, "moe_num_shared_experts", None) self.moe_num_shared_experts = getattr(config, "moe_num_shared_experts", 0)
self.ep_group = get_ep_group().device_group self.ep_group = get_ep_group().device_group
self.ep_size = self.ep_group.size() self.ep_size = self.ep_group.size()
self.n_routed_experts: int = config.moe_num_experts self.n_routed_experts: int = config.moe_num_experts
@@ -163,6 +163,7 @@ class Ernie4_5_MoeMoE(nn.Module):
torch.empty(config.moe_num_experts, dtype=torch.float32) torch.empty(config.moe_num_experts, dtype=torch.float32)
) )
self.shared_experts: Ernie4_5_MoeMLP | None
if self.has_shared_experts: if self.has_shared_experts:
intermediate_size = ( intermediate_size = (
config.moe_intermediate_size * config.moe_num_shared_experts config.moe_intermediate_size * config.moe_num_shared_experts
+27 -20
View File
@@ -1076,7 +1076,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
def _pixel_values_norm( def _pixel_values_norm(
self, self,
pixel_values: torch.Tensor, pixel_values: torch.Tensor,
mm_kwargs: object, mm_kwargs: Mapping[str, object],
) -> torch.Tensor: ) -> torch.Tensor:
hf_config = self.info.get_hf_config() hf_config = self.info.get_hf_config()
vision_config = hf_config.vision_config vision_config = hf_config.vision_config
@@ -1125,6 +1125,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
mm_data = dict(mm_data)
if "images" not in mm_data: if "images" not in mm_data:
mm_data["images"] = [] mm_data["images"] = []
if "videos" not in mm_data: if "videos" not in mm_data:
@@ -1136,15 +1137,15 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
hf_processor, "supports_video_metadata", False hf_processor, "supports_video_metadata", False
) )
if mm_data["videos"] and not supports_video_metadata: videos = mm_data["videos"]
assert isinstance(videos, Sequence)
if videos and not supports_video_metadata:
# Old HF processor, unwrap tuple to pure frames # Old HF processor, unwrap tuple to pure frames
logger.warning_once( logger.warning_once(
"HF processor doesn't support video metadata. " "HF processor doesn't support video metadata. "
"Timestamps will NOT be rendered. Please upgrade the model." "Timestamps will NOT be rendered. Please upgrade the model."
) )
mm_data["videos"] = [ mm_data["videos"] = [v[0] if isinstance(v, tuple) else v for v in videos]
v[0] if isinstance(v, tuple) else v for v in mm_data["videos"]
]
processor_output = self.info.ctx.call_hf_processor( processor_output = self.info.ctx.call_hf_processor(
hf_processor, hf_processor,
@@ -1286,6 +1287,7 @@ class Ernie4_5_VLDummyInputsBuilder(BaseDummyInputsBuilder[Ernie4_5_VLProcessing
image_overrides = mm_options.get("image") image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video") video_overrides = mm_options.get("video")
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
return { return {
"image": self._get_dummy_images( "image": self._get_dummy_images(
@@ -1541,10 +1543,14 @@ class Ernie4_5_VLMoeForConditionalGeneration(
offset = mm_feature.mm_position.offset offset = mm_feature.mm_position.offset
if mm_feature.modality == "image": if mm_feature.modality == "image":
t, h, w = mm_feature.data["image_grid_thw"].data.tolist() grid_thw = mm_feature.data["image_grid_thw"].data
assert isinstance(grid_thw, torch.Tensor)
t, h, w = grid_thw.tolist()
yield offset, t, h // spatial_conv_size, w // spatial_conv_size yield offset, t, h // spatial_conv_size, w // spatial_conv_size
elif mm_feature.modality == "video": elif mm_feature.modality == "video":
t, h, w = mm_feature.data["video_grid_thw"].data.tolist() grid_thw = mm_feature.data["video_grid_thw"].data
assert isinstance(grid_thw, torch.Tensor)
t, h, w = grid_thw.tolist()
yield ( yield (
offset, offset,
t // temporal_conv_size, t // temporal_conv_size,
@@ -1563,12 +1569,11 @@ class Ernie4_5_VLMoeForConditionalGeneration(
if pixel_values is None: if pixel_values is None:
return None return None
if pixel_values is not None: return Ernie4_5_VLImagePixelInputs(
return Ernie4_5_VLImagePixelInputs( type="pixel_values",
type="pixel_values", pixel_values=pixel_values,
pixel_values=pixel_values, image_grid_thw=image_grid_thw,
image_grid_thw=image_grid_thw, )
)
def _parse_and_validate_video_input( def _parse_and_validate_video_input(
self, **kwargs: object self, **kwargs: object
@@ -1579,12 +1584,11 @@ class Ernie4_5_VLMoeForConditionalGeneration(
if pixel_values_videos is None: if pixel_values_videos is None:
return None return None
if pixel_values_videos is not None: return Ernie4_5_VLVideoPixelInputs(
return Ernie4_5_VLVideoPixelInputs( type="pixel_values_videos",
type="pixel_values_videos", pixel_values_videos=pixel_values_videos,
pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw,
video_grid_thw=video_grid_thw, )
)
# -- SupportsEncoderCudaGraph protocol methods -- # -- SupportsEncoderCudaGraph protocol methods --
# #
@@ -1748,6 +1752,7 @@ class Ernie4_5_VLMoeForConditionalGeneration(
# the actual batch grid_thw, then scatter the post-merge embeddings. # the actual batch grid_thw, then scatter the post-merge embeddings.
# Ernie only uses the single "default" encoder path. # Ernie only uses the single "default" encoder path.
output = outputs["default"] output = outputs["default"]
assert batch_mm_kwargs is not None
grid_thw_cpu = batch_mm_kwargs["image_grid_thw"] grid_thw_cpu = batch_mm_kwargs["image_grid_thw"]
grid_thw = grid_thw_cpu.to(output.device, non_blocking=True) grid_thw = grid_thw_cpu.to(output.device, non_blocking=True)
# The valid token count slices the graph output for the eager # The valid token count slices the graph output for the eager
@@ -1799,7 +1804,9 @@ class Ernie4_5_VLMoeForConditionalGeneration(
return video_embeds.split(sizes.tolist()) return video_embeds.split(sizes.tolist())
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
modalities = {} modalities: dict[
str, Ernie4_5_VLImageInputs | Ernie4_5_VLVideoInputs | None
] = {}
# Preserve the order of modalities if there are multiple of them # Preserve the order of modalities if there are multiple of them
# from the order of kwargs. # from the order of kwargs.
+4 -2
View File
@@ -232,6 +232,7 @@ class Ernie4_5_VLMoeMoE(nn.Module):
assert text_moe_layer_start_index <= text_moe_layer_end_index assert text_moe_layer_start_index <= text_moe_layer_end_index
self.shared_experts: Ernie4_5_VLMoeMLP | None
if self.has_shared_experts: if self.has_shared_experts:
intermediate_size = ( intermediate_size = (
config.moe_intermediate_size[0] * config.moe_num_shared_experts config.moe_intermediate_size[0] * config.moe_num_shared_experts
@@ -771,9 +772,10 @@ class Ernie4_5_VLMoeForCausalLM(nn.Module, SupportsPP):
if is_pp_missing_parameter(name, self): if is_pp_missing_parameter(name, self):
continue continue
# Remapping the name of FP8 kv-scale. # Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict) remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if name is None: if remapped_name is None:
continue continue
name = remapped_name
param = params_dict[name] param = params_dict[name]
+1
View File
@@ -332,6 +332,7 @@ class Exaone4_5_ForConditionalGeneration(Qwen2_5_VLForConditionalGeneration):
config: Exaone4_5_Config = vllm_config.model_config.hf_config config: Exaone4_5_Config = vllm_config.model_config.hf_config
self.vllm_config = vllm_config self.vllm_config = vllm_config
multimodal_config = vllm_config.model_config.multimodal_config multimodal_config = vllm_config.model_config.multimodal_config
assert multimodal_config is not None
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data" self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
self.config = config self.config = config
+1 -8
View File
@@ -48,17 +48,11 @@ class Exaone4_5MultiTokenPredictor(ExaoneMoeMultiTokenPredictor):
model_config = vllm_config.model_config model_config = vllm_config.model_config
quant_config = vllm_config.quant_config quant_config = vllm_config.quant_config
lora_config = vllm_config.lora_config
config = model_config.hf_config config = model_config.hf_config
text_config = config.text_config text_config = config.text_config
self.config = config self.config = config
lora_vocab = ( self.vocab_size = config.vocab_size
(lora_config.lora_extra_vocab_size * (lora_config.max_loras or 1))
if lora_config
else 0
)
self.vocab_size = config.vocab_size + lora_vocab
self.org_vocab_size = config.vocab_size self.org_vocab_size = config.vocab_size
self.mtp_start_layer_idx = text_config.num_hidden_layers self.mtp_start_layer_idx = text_config.num_hidden_layers
@@ -67,7 +61,6 @@ class Exaone4_5MultiTokenPredictor(ExaoneMoeMultiTokenPredictor):
self.embed_tokens = VocabParallelEmbedding( self.embed_tokens = VocabParallelEmbedding(
self.vocab_size, self.vocab_size,
text_config.hidden_size, text_config.hidden_size,
org_num_embeddings=config.vocab_size,
) )
self.fc = ColumnParallelLinear( self.fc = ColumnParallelLinear(
+3 -20
View File
@@ -41,7 +41,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import ( from vllm.model_executor.layers.vocab_parallel_embedding import (
DEFAULT_VOCAB_PADDING_SIZE,
ParallelLMHead, ParallelLMHead,
VocabParallelEmbedding, VocabParallelEmbedding,
) )
@@ -112,6 +111,7 @@ class ExaoneMoe(nn.Module):
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
self.n_local_physical_experts = self.n_physical_experts // self.ep_size self.n_local_physical_experts = self.n_physical_experts // self.ep_size
self.shared_experts: ExaoneMoeGatedMLP | None
if getattr(config, "num_shared_experts", 0) > 0: if getattr(config, "num_shared_experts", 0) > 0:
intermediate_size = config.moe_intermediate_size * config.num_shared_experts intermediate_size = config.moe_intermediate_size * config.num_shared_experts
self.shared_experts = ExaoneMoeGatedMLP( self.shared_experts = ExaoneMoeGatedMLP(
@@ -292,7 +292,7 @@ class ExaoneMoeDecoderLayer(nn.Module):
config: PretrainedConfig, config: PretrainedConfig,
cache_config: CacheConfig | None = None, cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None, quant_config: QuantizationConfig | None = None,
is_mtp: bool = None, is_mtp: bool = False,
prefix: str = "", prefix: str = "",
) -> None: ) -> None:
super().__init__() super().__init__()
@@ -381,26 +381,19 @@ class ExaoneMoeModel(nn.Module):
config = vllm_config.model_config.hf_config config = vllm_config.model_config.hf_config
cache_config = vllm_config.cache_config cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config quant_config = vllm_config.quant_config
lora_config = vllm_config.lora_config
self.num_redundant_experts = ( self.num_redundant_experts = (
vllm_config.parallel_config.eplb_config.num_redundant_experts vllm_config.parallel_config.eplb_config.num_redundant_experts
) )
self.config = config self.config = config
self.quant_config = quant_config self.quant_config = quant_config
lora_vocab = ( self.vocab_size = config.vocab_size
(lora_config.lora_extra_vocab_size * (lora_config.max_loras or 1))
if lora_config
else 0
)
self.vocab_size = config.vocab_size + lora_vocab
if get_pp_group().is_first_rank or ( if get_pp_group().is_first_rank or (
config.tie_word_embeddings and get_pp_group().is_last_rank config.tie_word_embeddings and get_pp_group().is_last_rank
): ):
self.embed_tokens = VocabParallelEmbedding( self.embed_tokens = VocabParallelEmbedding(
self.vocab_size, self.vocab_size,
config.hidden_size, config.hidden_size,
org_num_embeddings=config.vocab_size,
quant_config=quant_config, quant_config=quant_config,
) )
else: else:
@@ -497,10 +490,8 @@ class ExaoneMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
super().__init__() super().__init__()
config = vllm_config.model_config.hf_config.get_text_config() config = vllm_config.model_config.hf_config.get_text_config()
quant_config = vllm_config.quant_config quant_config = vllm_config.quant_config
lora_config = vllm_config.lora_config
self.config = config self.config = config
self.lora_config = lora_config
self.quant_config = quant_config self.quant_config = quant_config
self.model = ExaoneMoeModel( self.model = ExaoneMoeModel(
@@ -509,17 +500,9 @@ class ExaoneMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
) )
if get_pp_group().is_last_rank: if get_pp_group().is_last_rank:
self.unpadded_vocab_size = config.vocab_size self.unpadded_vocab_size = config.vocab_size
if lora_config:
self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
self.lm_head = ParallelLMHead( self.lm_head = ParallelLMHead(
self.unpadded_vocab_size, self.unpadded_vocab_size,
config.hidden_size, config.hidden_size,
org_num_embeddings=config.vocab_size,
padding_size=DEFAULT_VOCAB_PADDING_SIZE
# We need bigger padding if using lora for kernel
# compatibility
if not lora_config
else lora_config.lora_vocab_padding_size,
quant_config=quant_config, quant_config=quant_config,
) )
if config.tie_word_embeddings: if config.tie_word_embeddings:
+1 -8
View File
@@ -46,16 +46,10 @@ class ExaoneMoeMultiTokenPredictor(nn.Module):
model_config = vllm_config.model_config model_config = vllm_config.model_config
quant_config = vllm_config.quant_config quant_config = vllm_config.quant_config
lora_config = vllm_config.lora_config
config = model_config.hf_config config = model_config.hf_config
self.config = config self.config = config
lora_vocab = ( self.vocab_size = config.vocab_size
(lora_config.lora_extra_vocab_size * (lora_config.max_loras or 1))
if lora_config
else 0
)
self.vocab_size = config.vocab_size + lora_vocab
self.org_vocab_size = config.vocab_size self.org_vocab_size = config.vocab_size
self.mtp_start_layer_idx = config.num_hidden_layers self.mtp_start_layer_idx = config.num_hidden_layers
@@ -64,7 +58,6 @@ class ExaoneMoeMultiTokenPredictor(nn.Module):
self.embed_tokens = VocabParallelEmbedding( self.embed_tokens = VocabParallelEmbedding(
self.vocab_size, self.vocab_size,
config.hidden_size, config.hidden_size,
org_num_embeddings=config.vocab_size,
) )
self.fc = ColumnParallelLinear( self.fc = ColumnParallelLinear(
@@ -231,7 +231,7 @@ class CacheOnlyAttentionLayer(nn.Module, AttentionLayerBase):
head_size: int, head_size: int,
cache_config: CacheConfig | None = None, cache_config: CacheConfig | None = None,
prefix: str = "", prefix: str = "",
attn_type: str = AttentionType.DECODER, attn_type: AttentionType = AttentionType.DECODER,
): ):
super().__init__() super().__init__()
@@ -329,7 +329,9 @@ class ExtractHiddenStatesModel(nn.Module):
super().__init__() super().__init__()
self.vllm_config = vllm_config self.vllm_config = vllm_config
self.hf_config = vllm_config.speculative_config.draft_model_config.hf_config speculative_config = vllm_config.speculative_config
assert speculative_config is not None
self.hf_config = speculative_config.draft_model_config.hf_config
self.hidden_size = vllm_config.model_config.get_hidden_size() self.hidden_size = vllm_config.model_config.get_hidden_size()
self.target_num_hidden_layers = ( self.target_num_hidden_layers = (
vllm_config.model_config.get_total_num_hidden_layers() vllm_config.model_config.get_total_num_hidden_layers()
+1
View File
@@ -413,6 +413,7 @@ class FalconModel(nn.Module):
else: else:
hidden_states = self.embed_input_ids(input_ids) hidden_states = self.embed_input_ids(input_ids)
else: else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"] hidden_states = intermediate_tensors["hidden_states"]
for layer in islice(self.h, self.start_layer, self.end_layer): for layer in islice(self.h, self.start_layer, self.end_layer):
hidden_states = layer(positions, hidden_states) hidden_states = layer(positions, hidden_states)
+2 -1
View File
@@ -12,7 +12,7 @@ from transformers import (
) )
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import MultiModalDataDict, PromptType from vllm.inputs import MultiModalDataDict, PromptType
from vllm.logger import init_logger from vllm.logger import init_logger
@@ -216,6 +216,7 @@ class FireRedASR2DummyInputsBuilder(BaseDummyInputsBuilder[FireRedASR2Processing
num_audios = mm_counts.get("audio", 0) num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio") audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
ret = { ret = {
"audio": self._get_dummy_audios( "audio": self._get_dummy_audios(
+8 -11
View File
@@ -18,14 +18,13 @@ from __future__ import annotations
from collections.abc import Iterable, Mapping, Sequence from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Literal from typing import Annotated, Literal
import numpy as np
import torch import torch
from torch import nn from torch import nn
from transformers import BatchFeature from transformers import BatchFeature
from vllm.config import ModelConfig, VllmConfig from vllm.config import ModelConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.speech_to_text import SpeechToTextConfig from vllm.config.speech_to_text import SpeechToTextConfig, SpeechToTextParams
from vllm.distributed import get_tensor_model_parallel_world_size from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict, PromptType from vllm.inputs import MultiModalDataDict, PromptType
from vllm.logger import init_logger from vllm.logger import init_logger
@@ -431,6 +430,7 @@ class FireRedLIDDummyInputsBuilder(BaseDummyInputsBuilder[FireRedLIDProcessingIn
audio_len = feature_extractor.chunk_length * sampling_rate audio_len = feature_extractor.chunk_length * sampling_rate
num_audios = mm_counts.get("audio", 0) num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio") audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
return { return {
"audio": self._get_dummy_audios( "audio": self._get_dummy_audios(
length=audio_len, length=audio_len,
@@ -747,13 +747,7 @@ class FireRedLIDForConditionalGeneration(
@classmethod @classmethod
def get_generation_prompt( def get_generation_prompt(
cls, cls,
audio: np.ndarray, stt_params: SpeechToTextParams,
stt_config: SpeechToTextConfig,
model_config: ModelConfig,
language: str | None,
task_type: Literal["transcribe", "translate"],
request_prompt: str,
to_language: str | None,
) -> PromptType: ) -> PromptType:
"""Build the prompt for the FireRedLID encoder-decoder model. """Build the prompt for the FireRedLID encoder-decoder model.
@@ -764,7 +758,10 @@ class FireRedLIDForConditionalGeneration(
"encoder_prompt": { "encoder_prompt": {
"prompt": "", "prompt": "",
"multi_modal_data": { "multi_modal_data": {
"audio": (audio, int(stt_config.sample_rate)), "audio": (
stt_params.audio,
int(stt_params.stt_config.sample_rate),
),
}, },
}, },
"decoder_prompt": { "decoder_prompt": {
+8 -5
View File
@@ -14,7 +14,7 @@ from transformers import (
) )
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams from vllm.config.speech_to_text import SpeechToTextParams
from vllm.distributed import get_tensor_model_parallel_world_size from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict, PromptType from vllm.inputs import MultiModalDataDict, PromptType
@@ -261,8 +261,8 @@ class SinusoidalPositionEncoder(torch.nn.Module):
def encode( def encode(
self, self,
positions: torch.Tensor = None, positions: torch.Tensor,
depth: int = None, depth: int,
dtype: torch.dtype = torch.float32, dtype: torch.dtype = torch.float32,
): ):
batch_size = positions.size(0) batch_size = positions.size(0)
@@ -573,8 +573,8 @@ class Transformer(nn.Module):
] ]
) )
def forward(self, hidden_states: torch.Tensor, ilens: int = 0): def forward(self, hidden_states: torch.Tensor, ilens: torch.Tensor):
max_len = max(ilens) max_len = ilens.max()
hidden_states = hidden_states[:, :max_len, :] hidden_states = hidden_states[:, :max_len, :]
batch_size, seq_len, dim = hidden_states.size() batch_size, seq_len, dim = hidden_states.size()
chunk_num = (seq_len - 1) // self.k + 1 chunk_num = (seq_len - 1) // self.k + 1
@@ -682,6 +682,8 @@ class FunASRModel(nn.Module):
self.feat_permute = False self.feat_permute = False
if self.feat_permute: if self.feat_permute:
if not isinstance(speech, torch.Tensor):
raise NotImplementedError("List audio inputs cannot be permuted")
encoder_out, encoder_out_lens = self.encoder.audio_encoder( encoder_out, encoder_out_lens = self.encoder.audio_encoder(
speech.permute(0, 2, 1), speech_lengths speech.permute(0, 2, 1), speech_lengths
) )
@@ -739,6 +741,7 @@ class FunASRDummyInputsBuilder(BaseDummyInputsBuilder[FunASRProcessingInfo]):
num_audios = mm_counts.get("audio", 0) num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio") audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
return { return {
"audio": self._get_dummy_audios( "audio": self._get_dummy_audios(
+3 -1
View File
@@ -26,7 +26,7 @@ from transformers.feature_extraction_utils import BatchFeature
from transformers.modeling_outputs import BaseModelOutput from transformers.modeling_outputs import BaseModelOutput
from vllm.config import VllmConfig from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.inputs import MultiModalDataDict from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention
from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear
@@ -600,6 +600,7 @@ class FunAudioChatDummyInputsBuilder(
num_audios = int(mm_counts.get("audio", 0)) num_audios = int(mm_counts.get("audio", 0))
audio_overrides = mm_options.get("audio") audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
return { return {
"audio": self._get_dummy_audios( "audio": self._get_dummy_audios(
length=audio_len, length=audio_len,
@@ -635,6 +636,7 @@ class FunAudioChatMultiModalProcessor(
processed_data = BatchFeature({"input_ids": input_ids}) processed_data = BatchFeature({"input_ids": input_ids})
processed_data.update(passthrough_data) processed_data.update(passthrough_data)
return processed_data return processed_data
assert isinstance(audios, Sequence)
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs) feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
sr = int(feature_extractor.sampling_rate) sr = int(feature_extractor.sampling_rate)