[Bugfix][Multimodal] Fix Qwen3-ASR audio preprocessing parity (#57146)

Signed-off-by: MatthewCYM <chenyiming0v0@gmail.com>
This commit is contained in:
MatthewCYM
2026-10-12 10:42:51 +00:00
committed by GitHub
parent 2534a96fcf
commit f7a3a7265f
4 changed files with 243 additions and 242 deletions
@@ -0,0 +1,181 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Qwen3-ASR preprocessing must preserve its HF processor's audio inputs."""
import numpy as np
import pytest
import torch
from transformers.feature_extraction_utils import BatchFeature
from transformers.models.qwen3_asr import Qwen3ASRProcessor
from vllm.config import ModelConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.inputs import batched_tensors_equal
from vllm.transformers_utils.processor import cached_processor_from_config
from .test_common import _assert_inputs_equal
_AUDIO_PROMPT = "<|audio_start|><|audio_pad|><|audio_end|>"
@pytest.fixture(params=["Qwen/Qwen3-ASR-0.6B", "Qwen/Qwen3-ASR-0.6B-hf"])
def asr_processor(request):
config = ModelConfig(
request.param,
dtype="float32",
max_model_len=4096,
limit_mm_per_prompt={"audio": 2},
mm_processor_cache_gb=0,
)
processor = MULTIMODAL_REGISTRY.create_processor(config)
native = Qwen3ASRProcessor.from_pretrained(request.param)
return processor, native
@pytest.mark.parametrize(
"samples", [321, 15999, 16000, 16001, 16159, 16160, 16161, 32001, 480001]
)
@pytest.mark.parametrize("num_audios", [1, 2])
def test_audio_processing_matches_hf(asr_processor, samples, num_audios):
"""Each clip must match HF independently of other clips in the request."""
processor, native = asr_processor
rng = np.random.RandomState(42)
audios = [
rng.randn(samples + index * 160).astype(np.float32)
for index in range(num_audios)
]
expected = [
native(text=_AUDIO_PROMPT, audio=audio, return_tensors="pt") for audio in audios
]
actual = processor(
_AUDIO_PROMPT * num_audios,
mm_items=processor.info.parse_mm_data({"audio": audios}),
hf_processor_mm_kwargs={},
)
expected_ids = [
token for item in expected for token in item["input_ids"][0].tolist()
]
assert actual["prompt_token_ids"] == expected_ids
masks = [item["input_features_mask"][0] for item in expected]
expected_features = torch.cat(
[
item["input_features"][0, :, mask.bool()]
for item, mask in zip(expected, masks)
],
dim=1,
)
actual_features = actual["mm_kwargs"].get_data()
torch.testing.assert_close(
actual_features["input_audio_features"], expected_features, rtol=0, atol=0
)
torch.testing.assert_close(
actual_features["audio_feature_lengths"],
torch.stack([mask.sum() for mask in masks]),
)
@pytest.mark.parametrize("cached_indices", [[], [0], [1], [0, 1]])
def test_audio_processing_cache_is_batch_independent(asr_processor, cached_indices):
"""Partial cache hits must not change features, masks, or prompt expansion."""
processor, _ = asr_processor
config = processor.info.ctx.model_config
config.get_multimodal_config().mm_processor_cache_gb = 1
cache = MultiModalProcessorOnlyCache(config)
rng = np.random.RandomState(42)
audios = [rng.randn(samples).astype(np.float32) for samples in (16001, 32000)]
for index in cached_indices:
processor(
_AUDIO_PROMPT,
mm_items=processor.info.parse_mm_data({"audio": audios[index]}),
hf_processor_mm_kwargs={},
cache=cache,
)
for batch in (audios, audios[::-1]):
kwargs = dict(
mm_items=processor.info.parse_mm_data({"audio": batch}),
hf_processor_mm_kwargs={},
)
_assert_inputs_equal(
processor(_AUDIO_PROMPT * 2, **kwargs),
processor(_AUDIO_PROMPT * 2, **kwargs, cache=cache),
)
def test_text_only_processing(asr_processor):
"""Empty media input must not invoke the audio feature extractor."""
processor, native = asr_processor
prompt = "Transcribe the audio."
actual = processor(
prompt,
mm_items=processor.info.parse_mm_data({}),
hf_processor_mm_kwargs={},
)
assert actual["prompt_token_ids"] == native.tokenizer.encode(prompt)
assert actual["mm_kwargs"].get_data() == {}
@pytest.mark.parametrize("hf_data", [{}, {"text": ""}, {"audio": None}, {"audio": []}])
def test_hf_processor_without_audio(asr_processor, hf_data):
"""The media hook must not require audio or call HF for absent waveforms."""
processor, _ = asr_processor
assert not processor._call_hf_processor(hf_data, {})
@pytest.mark.parametrize("hf_data", [{}, {"text": ""}])
def test_postprocessing_without_audio_features(asr_processor, hf_data):
"""Non-audio output must pass through even when HF input is nonempty."""
processor, _ = asr_processor
processed = BatchFeature({"input_ids": torch.tensor([[1, 2]])})
actual = processor._postprocess_hf_mm_data(hf_data, {}, processed)
assert actual is processed
assert set(actual) == {"input_ids"}
torch.testing.assert_close(actual["input_ids"], torch.tensor([[1, 2]]))
def test_chat_template_processing(asr_processor):
"""The generic processor loader must still render audio chat prompts."""
processor, native = asr_processor
chat_processor = cached_processor_from_config(processor.info.ctx.model_config)
messages = [
{"role": "system", "content": ""},
{"role": "user", "content": [{"type": "audio", "audio": "unused"}]},
]
prompt = chat_processor.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True
)
expected_prompt = native.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True
)
assert prompt == expected_prompt
audio = np.random.RandomState(42).randn(16001).astype(np.float32)
expected = native(text=expected_prompt, audio=audio, return_tensors="pt")
actual = processor(
prompt,
mm_items=processor.info.parse_mm_data({"audio": audio}),
hf_processor_mm_kwargs={},
)
assert actual["prompt_token_ids"] == expected["input_ids"][0].tolist()
def test_precomputed_audio_features(asr_processor):
"""Precomputed features must bypass waveform extraction unchanged."""
processor, _ = asr_processor
processor.info.ctx.get_mm_config().enable_mm_embeds = True
audio = np.random.RandomState(42).randn(16001).astype(np.float32)
expected = processor(
_AUDIO_PROMPT,
mm_items=processor.info.parse_mm_data({"audio": audio}),
hf_processor_mm_kwargs={},
)
features = expected["mm_kwargs"].get_data()
actual = processor(
_AUDIO_PROMPT,
mm_items=processor.info.parse_mm_data({"audio": features}),
hf_processor_mm_kwargs={},
)
assert actual["prompt_token_ids"] == expected["prompt_token_ids"]
assert actual["mm_placeholders"] == expected["mm_placeholders"]
assert batched_tensors_equal(actual["mm_kwargs"].get_data(), features)
+62 -11
View File
@@ -29,6 +29,7 @@ import regex as re
import torch
import torch.nn as nn
from transformers.feature_extraction_utils import BatchFeature
from transformers.models.qwen3_asr import Qwen3ASRFeatureExtractor, Qwen3ASRProcessor
from transformers.models.whisper import WhisperFeatureExtractor
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
@@ -54,7 +55,6 @@ from vllm.model_executor.models.qwen2_5_omni_thinker import (
from vllm.model_executor.models.qwen3 import Qwen3ForCausalLM
from vllm.model_executor.models.qwen3_omni_moe_thinker import (
Qwen3OmniMoeAudioEncoder,
Qwen3OmniMoeThinkerMultiModalProcessor,
)
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
@@ -80,6 +80,7 @@ from vllm.multimodal.parse import (
)
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseMultiModalProcessor,
BaseProcessingInfo,
PromptReplacement,
PromptUpdate,
@@ -92,9 +93,6 @@ from vllm.transformers_utils.configs.qwen3_asr import (
Qwen3ASRThinkerConfig,
)
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.transformers_utils.processors.qwen3_asr import (
Qwen3ASRProcessor,
)
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.torch_utils import async_tensor_h2d
@@ -188,18 +186,19 @@ class Qwen3ASRProcessingInfo(BaseProcessingInfo):
return self.ctx.get_hf_config(Qwen3ASRConfig).thinker_config
def get_hf_processor(self, **kwargs: object) -> Qwen3ASRProcessor:
processor = self.ctx.get_hf_processor(
return self.ctx.get_hf_processor(
Qwen3ASRProcessor,
**kwargs,
)
if not hasattr(processor, "audio_token"):
processor.audio_token = "<|audio_pad|>"
return processor
def get_feature_extractor(self, **kwargs: object) -> WhisperFeatureExtractor:
def get_feature_extractor(
self, **kwargs: object
) -> WhisperFeatureExtractor | Qwen3ASRFeatureExtractor:
hf_processor = self.get_hf_processor(**kwargs)
feature_extractor = hf_processor.feature_extractor
assert isinstance(feature_extractor, WhisperFeatureExtractor)
assert isinstance(
feature_extractor, (WhisperFeatureExtractor, Qwen3ASRFeatureExtractor)
)
return feature_extractor
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
@@ -275,8 +274,60 @@ class Qwen3ASRMultiModalDataParser(MultiModalDataParser):
class Qwen3ASRMultiModalProcessor(
Qwen3OmniMoeThinkerMultiModalProcessor[Qwen3ASRProcessingInfo]
BaseMultiModalProcessor[Qwen3ASRProcessingInfo],
):
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _call_hf_processor(
self,
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
audios = hf_data.get("audio")
if not audios:
return BatchFeature()
assert isinstance(audios, list)
if len(audios) == 1:
return super()._call_hf_processor(hf_data, hf_kwargs)
# HF requires one text per audio, and batch padding changes clip features.
# Keep clips independent so partial cache hits cannot change their inputs.
text = self._get_hf_mm_text({"audio": 1})
outputs: dict[str, list[torch.Tensor]] = {
"input_features": [],
"input_features_mask": [],
}
for audio in audios:
processed = super()._call_hf_processor(
dict(hf_data, text=text, audio=audio), hf_kwargs
)
for key, values in outputs.items():
values.append(processed[key][0])
return BatchFeature(outputs)
def _postprocess_hf_mm_data(
self,
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "input_features" not in processed_data:
return processed_data
features = processed_data.pop("input_features")
masks = processed_data.pop("input_features_mask")
processed_data["input_audio_features"] = torch.cat(
[feature[:, mask.bool()] for feature, mask in zip(features, masks)], dim=1
)
processed_data["feature_attention_mask"] = masks
processed_data["audio_feature_lengths"] = torch.stack(
[mask.sum() for mask in masks]
)
return processed_data
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
@@ -38,7 +38,6 @@ __all__ = [
"OpenVLAProcessor",
"OvisProcessor",
"Ovis2_5Processor",
"Qwen3ASRProcessor",
"Step3VLProcessor",
"InklingProcessor",
"InklingImageProcessor",
@@ -74,7 +73,6 @@ _CLASS_TO_MODULE: dict[str, str] = {
"OpenVLAProcessor": "vllm.transformers_utils.processors.openvla",
"OvisProcessor": "vllm.transformers_utils.processors.ovis",
"Ovis2_5Processor": "vllm.transformers_utils.processors.ovis2_5",
"Qwen3ASRProcessor": "vllm.transformers_utils.processors.qwen3_asr",
"Step3VLProcessor": "vllm.transformers_utils.processors.step3_vl",
"InklingProcessor": "vllm.transformers_utils.processors.inkling",
"InklingImageProcessor": "vllm.transformers_utils.processors.inkling",
@@ -1,229 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa
# mypy: ignore-errors
# coding=utf-8
# Copyright 2026 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import regex as re
import numpy as np
from transformers import AutoProcessor
from transformers.audio_utils import AudioInput
from transformers.feature_extraction_utils import BatchFeature
from transformers.processing_utils import ProcessingKwargs, ProcessorMixin
from transformers.tokenization_utils_base import TextInput
class Qwen3ASRProcessorKwargs(ProcessingKwargs, total=False):
_defaults = {
"text_kwargs": {
"padding": False,
"padding_side": "left",
},
"audio_kwargs": {
"sampling_rate": 16000,
"padding": True,
"return_attention_mask": True,
},
}
def _get_feat_extract_output_lengths(input_lengths):
"""
Computes the output length of the convolutional layers and the output length of the audio encoder
"""
input_lengths_leave = input_lengths % 100
feat_lengths = (input_lengths_leave - 1) // 2 + 1
output_lengths = (
((feat_lengths - 1) // 2 + 1 - 1) // 2 + 1 + (input_lengths // 100) * 13
)
return output_lengths
class Qwen3ASRProcessor(ProcessorMixin):
r"""
Constructs a Qwen3ASR processor.
[`Qwen3ASRProcessor`] offers all the functionalities of [`WhisperFeatureExtractor`], and [`Qwen2TokenizerFast`]. See the
[`~Qwen3ASRProcessor.__call__`] and [`~Qwen3ASRProcessor.decode`] for more information.
Args:
feature_extractor ([`WhisperFeatureExtractor`], *optional*):
The audio feature extractor.
tokenizer ([`Qwen2TokenizerFast`], *optional*):
The text tokenizer.
chat_template (`Optional[str]`, *optional*):
The Jinja template to use for formatting the conversation. If not provided, the default chat template is used.
"""
attributes = ["feature_extractor", "tokenizer"]
feature_extractor_class = "WhisperFeatureExtractor"
tokenizer_class = ("Qwen2Tokenizer", "Qwen2TokenizerFast")
def __init__(self, feature_extractor=None, tokenizer=None, chat_template=None):
super().__init__(feature_extractor, tokenizer, chat_template=chat_template)
self.audio_token = self.tokenizer.audio_token
self.audio_bos_token = self.tokenizer.audio_bos_token
self.audio_eos_token = self.tokenizer.audio_eos_token
def __call__(
self,
text: TextInput = None,
audio: AudioInput = None,
**kwargs,
) -> BatchFeature:
"""
Main method to prepare for the model one or several sequences(s) and audio(s). This method forwards the `text`
and `kwargs` arguments to Qwen2TokenizerFast's [`~Qwen2TokenizerFast.__call__`] if `text` is not `None` to encode
the text. To prepare the audio(s), this method forwards the `audio` and `kwargs` arguments to
WhisperFeatureExtractor's [`~WhisperFeatureExtractor.__call__`] if `audio` is not `None`. Please refer to the doctsring
of the above two methods for more information.
Args:
text (`str`, `List[str]`, `List[List[str]]`):
The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
(pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
`is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
audio (`np.ndarray`, `List[np.ndarray]`):
The audio or batch of audio to be prepared. Each audio can be a NumPy array.
"""
if text is None:
raise ValueError("You need to specify either a `text` input to process.")
output_kwargs = self._merge_kwargs(
Qwen3ASRProcessorKwargs,
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
**kwargs,
)
if audio is not None:
output_kwargs["audio_kwargs"]["padding"] = True
output_kwargs["audio_kwargs"]["truncation"] = False
audio_inputs = self.feature_extractor(
audio, **output_kwargs["audio_kwargs"]
)
audio_inputs["feature_attention_mask"] = audio_inputs.pop(
"attention_mask"
) # rename feature_attention_mask to prevent conflicts later on
audio_inputs["input_features"] = audio_inputs.pop(
"input_features"
) # rename input_features to prevent conflicts later on
audio_lengths = iter(
_get_feat_extract_output_lengths(
audio_inputs["feature_attention_mask"].sum(-1)
)
)
else:
audio_inputs = {}
audio_lengths = iter([])
if not isinstance(text, list):
text = [text]
text = self.replace_multimodal_special_tokens(
text,
audio_lengths,
)
texts_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
return BatchFeature(
data={**texts_inputs, **audio_inputs},
tensor_type=kwargs.get("return_tensors"),
)
def replace_multimodal_special_tokens(
self,
text,
audio_lengths,
):
processed_text = []
for sample in text:
positions = []
special_tokens = [re.escape(tok) for tok in [self.audio_token]]
pattern = "|".join(special_tokens)
positions = sorted(
[
(match.start(), match.group())
for match in re.finditer(pattern, sample)
]
)
positions.sort(key=lambda x: x[0])
for _, special_token in positions:
if special_token == self.audio_token:
sample = sample.replace(
self.audio_token,
"<|audio_placeholder|>" * next(audio_lengths),
1,
)
sample = sample.replace("<|audio_placeholder|>", self.audio_token)
processed_text.append(sample)
return processed_text
def get_chunked_index(
self, token_indices: np.ndarray, tokens_per_chunk: int
) -> list[tuple[int, int]]:
"""
Splits token index list into chunks based on token value ranges.
Given a list of token indices, returns a list of (start, end) index tuples representing
slices of the list where the token values fall within successive ranges of `tokens_per_chunk`.
For example, if `tokens_per_chunk` is 1000, the function will create chunks such that:
- the first chunk contains token values < 1000,
- the second chunk contains values >= 1000 and < 2000, and so on.
Parameters:
token_indices (`np.ndarray`): A monotonically increasing list of token index values.
tokens_per_chunk (`int`): Number of tokens per chunk (used as the chunk size threshold).
Returns:
`list[tuple[int, int]]`: A list of tuples, each representing the start (inclusive)
and end (exclusive) indices of a chunk in `token_indices`.
"""
def _iter():
i, start_idx = 0, 0 # skip bos token
current_chunk = 1
while i < len(token_indices): # skip eos token
if token_indices[i] >= current_chunk * tokens_per_chunk:
yield (start_idx, i)
start_idx = i
current_chunk += 1
i += 1
yield (start_idx, len(token_indices))
return list(_iter())
def apply_chat_template(self, conversations, chat_template=None, **kwargs):
kwargs["return_dict"] = False
return super().apply_chat_template(conversations, chat_template, **kwargs)
@property
def model_input_names(self):
tokenizer_input_names = self.tokenizer.model_input_names
feature_extractor_input_names = self.feature_extractor.model_input_names
return list(
dict.fromkeys(
tokenizer_input_names
+ feature_extractor_input_names
+ ["feature_attention_mask"]
)
)