[Rust Frontend] Integrate MM audio support (#48554)

Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Bugen Zhao
2026-07-15 15:00:17 +08:00
committed by GitHub
co-authored by Isotr0py mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent adce068118
commit e281ac663a
18 changed files with 1275 additions and 258 deletions
@@ -114,6 +114,23 @@ except (ImportError, ModuleNotFoundError):
logger = init_logger(__name__)
def unpad_and_flat_audio_features(
input_audio_features: torch.Tensor,
audio_feature_lengths: torch.Tensor,
) -> torch.Tensor:
"""Unpad and flatten batched audio features."""
if len(audio_feature_lengths) != input_audio_features.size(0):
raise ValueError(
"Length of audio_feature_lengths must match "
"the batch size of input_audio_features."
)
unpadded_features = [
audio_feature[..., :length]
for audio_feature, length in zip(input_audio_features, audio_feature_lengths)
]
return torch.cat(unpadded_features, dim=-1)
def check_interleaved_audio_video(
is_video: torch.Tensor,
is_audio: torch.Tensor,
@@ -887,6 +904,17 @@ class Qwen2_5OmniConditionalGenerationMixin:
if input_audio_features is None:
return None
# inputs features from rust frontend is batched and padded
# with shape [batch_size, n_mels, padded_seq_len], different
# from python's shape [n_mels, batch_size * seq_len]
if (
isinstance(input_audio_features, torch.Tensor)
and input_audio_features.dim() == 3
):
input_audio_features = unpad_and_flat_audio_features(
input_audio_features, audio_feature_lengths
)
return Qwen2_5OmniAudioFeatureInputs(
type="audio_features",
input_features=input_audio_features,
+20 -1
View File
@@ -46,9 +46,12 @@ from vllm.model_executor.models.interfaces import (
SupportsTranscription,
)
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.model_executor.models.qwen2_5_omni_thinker import (
Qwen2_5OmniAudioFeatureInputs,
unpad_and_flat_audio_features,
)
from vllm.model_executor.models.qwen3 import Qwen3ForCausalLM
from vllm.model_executor.models.qwen3_omni_moe_thinker import (
Qwen2_5OmniAudioFeatureInputs,
Qwen3OmniMoeAudioEncoder,
Qwen3OmniMoeThinkerMultiModalProcessor,
)
@@ -363,6 +366,11 @@ class Qwen3ASRForConditionalGeneration(
"thinker.lm_head.": "language_model.lm_head.",
"thinker.model.": "language_model.model.",
"thinker.": "",
# HF format mapper
"model.audio_tower.": "audio_tower.",
"model.language_model.": "language_model.model.",
"model.multi_modal_projector.linear_1.": "audio_tower.proj1.",
"model.multi_modal_projector.linear_2.": "audio_tower.proj2.",
}
)
@@ -412,6 +420,17 @@ class Qwen3ASRForConditionalGeneration(
if input_audio_features is None:
return None
# inputs features from rust frontend is batched and padded
# with shape [batch_size, n_mels, padded_seq_len], different
# from python's shape [n_mels, batch_size * seq_len]
if (
isinstance(input_audio_features, torch.Tensor)
and input_audio_features.dim() == 3
):
input_audio_features = unpad_and_flat_audio_features(
input_audio_features, audio_feature_lengths
)
return Qwen2_5OmniAudioFeatureInputs(
type="audio_features",
input_features=input_audio_features,