[Bugfix][Multimodal] Fix Qwen3-ASR audio preprocessing parity (#57146)

Signed-off-by: MatthewCYM <chenyiming0v0@gmail.com>
This commit is contained in:
MatthewCYM
2026-10-12 10:42:51 +00:00
committed by GitHub
parent 2534a96fcf
commit f7a3a7265f
4 changed files with 243 additions and 242 deletions
+62 -11
View File
@@ -29,6 +29,7 @@ import regex as re
import torch
import torch.nn as nn
from transformers.feature_extraction_utils import BatchFeature
from transformers.models.qwen3_asr import Qwen3ASRFeatureExtractor, Qwen3ASRProcessor
from transformers.models.whisper import WhisperFeatureExtractor
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
@@ -54,7 +55,6 @@ from vllm.model_executor.models.qwen2_5_omni_thinker import (
from vllm.model_executor.models.qwen3 import Qwen3ForCausalLM
from vllm.model_executor.models.qwen3_omni_moe_thinker import (
Qwen3OmniMoeAudioEncoder,
Qwen3OmniMoeThinkerMultiModalProcessor,
)
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
@@ -80,6 +80,7 @@ from vllm.multimodal.parse import (
)
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseMultiModalProcessor,
BaseProcessingInfo,
PromptReplacement,
PromptUpdate,
@@ -92,9 +93,6 @@ from vllm.transformers_utils.configs.qwen3_asr import (
Qwen3ASRThinkerConfig,
)
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.transformers_utils.processors.qwen3_asr import (
Qwen3ASRProcessor,
)
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.torch_utils import async_tensor_h2d
@@ -188,18 +186,19 @@ class Qwen3ASRProcessingInfo(BaseProcessingInfo):
return self.ctx.get_hf_config(Qwen3ASRConfig).thinker_config
def get_hf_processor(self, **kwargs: object) -> Qwen3ASRProcessor:
processor = self.ctx.get_hf_processor(
return self.ctx.get_hf_processor(
Qwen3ASRProcessor,
**kwargs,
)
if not hasattr(processor, "audio_token"):
processor.audio_token = "<|audio_pad|>"
return processor
def get_feature_extractor(self, **kwargs: object) -> WhisperFeatureExtractor:
def get_feature_extractor(
self, **kwargs: object
) -> WhisperFeatureExtractor | Qwen3ASRFeatureExtractor:
hf_processor = self.get_hf_processor(**kwargs)
feature_extractor = hf_processor.feature_extractor
assert isinstance(feature_extractor, WhisperFeatureExtractor)
assert isinstance(
feature_extractor, (WhisperFeatureExtractor, Qwen3ASRFeatureExtractor)
)
return feature_extractor
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
@@ -275,8 +274,60 @@ class Qwen3ASRMultiModalDataParser(MultiModalDataParser):
class Qwen3ASRMultiModalProcessor(
Qwen3OmniMoeThinkerMultiModalProcessor[Qwen3ASRProcessingInfo]
BaseMultiModalProcessor[Qwen3ASRProcessingInfo],
):
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _call_hf_processor(
self,
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
audios = hf_data.get("audio")
if not audios:
return BatchFeature()
assert isinstance(audios, list)
if len(audios) == 1:
return super()._call_hf_processor(hf_data, hf_kwargs)
# HF requires one text per audio, and batch padding changes clip features.
# Keep clips independent so partial cache hits cannot change their inputs.
text = self._get_hf_mm_text({"audio": 1})
outputs: dict[str, list[torch.Tensor]] = {
"input_features": [],
"input_features_mask": [],
}
for audio in audios:
processed = super()._call_hf_processor(
dict(hf_data, text=text, audio=audio), hf_kwargs
)
for key, values in outputs.items():
values.append(processed[key][0])
return BatchFeature(outputs)
def _postprocess_hf_mm_data(
self,
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "input_features" not in processed_data:
return processed_data
features = processed_data.pop("input_features")
masks = processed_data.pop("input_features_mask")
processed_data["input_audio_features"] = torch.cat(
[feature[:, mask.bool()] for feature, mask in zip(features, masks)], dim=1
)
processed_data["feature_attention_mask"] = masks
processed_data["audio_feature_lengths"] = torch.stack(
[mask.sum() for mask in masks]
)
return processed_data
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,