[Rust Frontend] Integrate MM audio support (#48554)
Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
Isotr0py
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
adce068118
commit
e281ac663a
@@ -114,6 +114,23 @@ except (ImportError, ModuleNotFoundError):
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def unpad_and_flat_audio_features(
|
||||
input_audio_features: torch.Tensor,
|
||||
audio_feature_lengths: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Unpad and flatten batched audio features."""
|
||||
if len(audio_feature_lengths) != input_audio_features.size(0):
|
||||
raise ValueError(
|
||||
"Length of audio_feature_lengths must match "
|
||||
"the batch size of input_audio_features."
|
||||
)
|
||||
unpadded_features = [
|
||||
audio_feature[..., :length]
|
||||
for audio_feature, length in zip(input_audio_features, audio_feature_lengths)
|
||||
]
|
||||
return torch.cat(unpadded_features, dim=-1)
|
||||
|
||||
|
||||
def check_interleaved_audio_video(
|
||||
is_video: torch.Tensor,
|
||||
is_audio: torch.Tensor,
|
||||
@@ -887,6 +904,17 @@ class Qwen2_5OmniConditionalGenerationMixin:
|
||||
if input_audio_features is None:
|
||||
return None
|
||||
|
||||
# inputs features from rust frontend is batched and padded
|
||||
# with shape [batch_size, n_mels, padded_seq_len], different
|
||||
# from python's shape [n_mels, batch_size * seq_len]
|
||||
if (
|
||||
isinstance(input_audio_features, torch.Tensor)
|
||||
and input_audio_features.dim() == 3
|
||||
):
|
||||
input_audio_features = unpad_and_flat_audio_features(
|
||||
input_audio_features, audio_feature_lengths
|
||||
)
|
||||
|
||||
return Qwen2_5OmniAudioFeatureInputs(
|
||||
type="audio_features",
|
||||
input_features=input_audio_features,
|
||||
|
||||
@@ -46,9 +46,12 @@ from vllm.model_executor.models.interfaces import (
|
||||
SupportsTranscription,
|
||||
)
|
||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.model_executor.models.qwen2_5_omni_thinker import (
|
||||
Qwen2_5OmniAudioFeatureInputs,
|
||||
unpad_and_flat_audio_features,
|
||||
)
|
||||
from vllm.model_executor.models.qwen3 import Qwen3ForCausalLM
|
||||
from vllm.model_executor.models.qwen3_omni_moe_thinker import (
|
||||
Qwen2_5OmniAudioFeatureInputs,
|
||||
Qwen3OmniMoeAudioEncoder,
|
||||
Qwen3OmniMoeThinkerMultiModalProcessor,
|
||||
)
|
||||
@@ -363,6 +366,11 @@ class Qwen3ASRForConditionalGeneration(
|
||||
"thinker.lm_head.": "language_model.lm_head.",
|
||||
"thinker.model.": "language_model.model.",
|
||||
"thinker.": "",
|
||||
# HF format mapper
|
||||
"model.audio_tower.": "audio_tower.",
|
||||
"model.language_model.": "language_model.model.",
|
||||
"model.multi_modal_projector.linear_1.": "audio_tower.proj1.",
|
||||
"model.multi_modal_projector.linear_2.": "audio_tower.proj2.",
|
||||
}
|
||||
)
|
||||
|
||||
@@ -412,6 +420,17 @@ class Qwen3ASRForConditionalGeneration(
|
||||
if input_audio_features is None:
|
||||
return None
|
||||
|
||||
# inputs features from rust frontend is batched and padded
|
||||
# with shape [batch_size, n_mels, padded_seq_len], different
|
||||
# from python's shape [n_mels, batch_size * seq_len]
|
||||
if (
|
||||
isinstance(input_audio_features, torch.Tensor)
|
||||
and input_audio_features.dim() == 3
|
||||
):
|
||||
input_audio_features = unpad_and_flat_audio_features(
|
||||
input_audio_features, audio_feature_lengths
|
||||
)
|
||||
|
||||
return Qwen2_5OmniAudioFeatureInputs(
|
||||
type="audio_features",
|
||||
input_features=input_audio_features,
|
||||
|
||||
Reference in New Issue
Block a user