[Bugfix][Multimodal] Fix Qwen3-ASR audio preprocessing parity (#57146)
Signed-off-by: MatthewCYM <chenyiming0v0@gmail.com>
This commit is contained in:
@@ -29,6 +29,7 @@ import regex as re
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
from transformers.models.qwen3_asr import Qwen3ASRFeatureExtractor, Qwen3ASRProcessor
|
||||
from transformers.models.whisper import WhisperFeatureExtractor
|
||||
|
||||
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
||||
@@ -54,7 +55,6 @@ from vllm.model_executor.models.qwen2_5_omni_thinker import (
|
||||
from vllm.model_executor.models.qwen3 import Qwen3ForCausalLM
|
||||
from vllm.model_executor.models.qwen3_omni_moe_thinker import (
|
||||
Qwen3OmniMoeAudioEncoder,
|
||||
Qwen3OmniMoeThinkerMultiModalProcessor,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
@@ -80,6 +80,7 @@ from vllm.multimodal.parse import (
|
||||
)
|
||||
from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
@@ -92,9 +93,6 @@ from vllm.transformers_utils.configs.qwen3_asr import (
|
||||
Qwen3ASRThinkerConfig,
|
||||
)
|
||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||
from vllm.transformers_utils.processors.qwen3_asr import (
|
||||
Qwen3ASRProcessor,
|
||||
)
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
@@ -188,18 +186,19 @@ class Qwen3ASRProcessingInfo(BaseProcessingInfo):
|
||||
return self.ctx.get_hf_config(Qwen3ASRConfig).thinker_config
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> Qwen3ASRProcessor:
|
||||
processor = self.ctx.get_hf_processor(
|
||||
return self.ctx.get_hf_processor(
|
||||
Qwen3ASRProcessor,
|
||||
**kwargs,
|
||||
)
|
||||
if not hasattr(processor, "audio_token"):
|
||||
processor.audio_token = "<|audio_pad|>"
|
||||
return processor
|
||||
|
||||
def get_feature_extractor(self, **kwargs: object) -> WhisperFeatureExtractor:
|
||||
def get_feature_extractor(
|
||||
self, **kwargs: object
|
||||
) -> WhisperFeatureExtractor | Qwen3ASRFeatureExtractor:
|
||||
hf_processor = self.get_hf_processor(**kwargs)
|
||||
feature_extractor = hf_processor.feature_extractor
|
||||
assert isinstance(feature_extractor, WhisperFeatureExtractor)
|
||||
assert isinstance(
|
||||
feature_extractor, (WhisperFeatureExtractor, Qwen3ASRFeatureExtractor)
|
||||
)
|
||||
return feature_extractor
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
@@ -275,8 +274,60 @@ class Qwen3ASRMultiModalDataParser(MultiModalDataParser):
|
||||
|
||||
|
||||
class Qwen3ASRMultiModalProcessor(
|
||||
Qwen3OmniMoeThinkerMultiModalProcessor[Qwen3ASRProcessingInfo]
|
||||
BaseMultiModalProcessor[Qwen3ASRProcessingInfo],
|
||||
):
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
hf_data: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
audios = hf_data.get("audio")
|
||||
if not audios:
|
||||
return BatchFeature()
|
||||
assert isinstance(audios, list)
|
||||
|
||||
if len(audios) == 1:
|
||||
return super()._call_hf_processor(hf_data, hf_kwargs)
|
||||
|
||||
# HF requires one text per audio, and batch padding changes clip features.
|
||||
# Keep clips independent so partial cache hits cannot change their inputs.
|
||||
text = self._get_hf_mm_text({"audio": 1})
|
||||
outputs: dict[str, list[torch.Tensor]] = {
|
||||
"input_features": [],
|
||||
"input_features_mask": [],
|
||||
}
|
||||
for audio in audios:
|
||||
processed = super()._call_hf_processor(
|
||||
dict(hf_data, text=text, audio=audio), hf_kwargs
|
||||
)
|
||||
for key, values in outputs.items():
|
||||
values.append(processed[key][0])
|
||||
|
||||
return BatchFeature(outputs)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
hf_data: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if "input_features" not in processed_data:
|
||||
return processed_data
|
||||
|
||||
features = processed_data.pop("input_features")
|
||||
masks = processed_data.pop("input_features_mask")
|
||||
processed_data["input_audio_features"] = torch.cat(
|
||||
[feature[:, mask.bool()] for feature, mask in zip(features, masks)], dim=1
|
||||
)
|
||||
processed_data["feature_attention_mask"] = masks
|
||||
processed_data["audio_feature_lengths"] = torch.stack(
|
||||
[mask.sum() for mask in masks]
|
||||
)
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
|
||||
Reference in New Issue
Block a user