diff --git a/tests/models/multimodal/processing/test_qwen3_asr.py b/tests/models/multimodal/processing/test_qwen3_asr.py new file mode 100644 index 0000000000..b016a9d708 --- /dev/null +++ b/tests/models/multimodal/processing/test_qwen3_asr.py @@ -0,0 +1,181 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Qwen3-ASR preprocessing must preserve its HF processor's audio inputs.""" + +import numpy as np +import pytest +import torch +from transformers.feature_extraction_utils import BatchFeature +from transformers.models.qwen3_asr import Qwen3ASRProcessor + +from vllm.config import ModelConfig +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.cache import MultiModalProcessorOnlyCache +from vllm.multimodal.inputs import batched_tensors_equal +from vllm.transformers_utils.processor import cached_processor_from_config + +from .test_common import _assert_inputs_equal + +_AUDIO_PROMPT = "<|audio_start|><|audio_pad|><|audio_end|>" + + +@pytest.fixture(params=["Qwen/Qwen3-ASR-0.6B", "Qwen/Qwen3-ASR-0.6B-hf"]) +def asr_processor(request): + config = ModelConfig( + request.param, + dtype="float32", + max_model_len=4096, + limit_mm_per_prompt={"audio": 2}, + mm_processor_cache_gb=0, + ) + processor = MULTIMODAL_REGISTRY.create_processor(config) + native = Qwen3ASRProcessor.from_pretrained(request.param) + return processor, native + + +@pytest.mark.parametrize( + "samples", [321, 15999, 16000, 16001, 16159, 16160, 16161, 32001, 480001] +) +@pytest.mark.parametrize("num_audios", [1, 2]) +def test_audio_processing_matches_hf(asr_processor, samples, num_audios): + """Each clip must match HF independently of other clips in the request.""" + processor, native = asr_processor + rng = np.random.RandomState(42) + audios = [ + rng.randn(samples + index * 160).astype(np.float32) + for index in range(num_audios) + ] + expected = [ + native(text=_AUDIO_PROMPT, audio=audio, return_tensors="pt") for audio in audios + ] + actual = processor( + _AUDIO_PROMPT * num_audios, + mm_items=processor.info.parse_mm_data({"audio": audios}), + hf_processor_mm_kwargs={}, + ) + + expected_ids = [ + token for item in expected for token in item["input_ids"][0].tolist() + ] + assert actual["prompt_token_ids"] == expected_ids + masks = [item["input_features_mask"][0] for item in expected] + expected_features = torch.cat( + [ + item["input_features"][0, :, mask.bool()] + for item, mask in zip(expected, masks) + ], + dim=1, + ) + actual_features = actual["mm_kwargs"].get_data() + torch.testing.assert_close( + actual_features["input_audio_features"], expected_features, rtol=0, atol=0 + ) + torch.testing.assert_close( + actual_features["audio_feature_lengths"], + torch.stack([mask.sum() for mask in masks]), + ) + + +@pytest.mark.parametrize("cached_indices", [[], [0], [1], [0, 1]]) +def test_audio_processing_cache_is_batch_independent(asr_processor, cached_indices): + """Partial cache hits must not change features, masks, or prompt expansion.""" + processor, _ = asr_processor + config = processor.info.ctx.model_config + config.get_multimodal_config().mm_processor_cache_gb = 1 + cache = MultiModalProcessorOnlyCache(config) + rng = np.random.RandomState(42) + audios = [rng.randn(samples).astype(np.float32) for samples in (16001, 32000)] + for index in cached_indices: + processor( + _AUDIO_PROMPT, + mm_items=processor.info.parse_mm_data({"audio": audios[index]}), + hf_processor_mm_kwargs={}, + cache=cache, + ) + + for batch in (audios, audios[::-1]): + kwargs = dict( + mm_items=processor.info.parse_mm_data({"audio": batch}), + hf_processor_mm_kwargs={}, + ) + _assert_inputs_equal( + processor(_AUDIO_PROMPT * 2, **kwargs), + processor(_AUDIO_PROMPT * 2, **kwargs, cache=cache), + ) + + +def test_text_only_processing(asr_processor): + """Empty media input must not invoke the audio feature extractor.""" + processor, native = asr_processor + prompt = "Transcribe the audio." + actual = processor( + prompt, + mm_items=processor.info.parse_mm_data({}), + hf_processor_mm_kwargs={}, + ) + assert actual["prompt_token_ids"] == native.tokenizer.encode(prompt) + assert actual["mm_kwargs"].get_data() == {} + + +@pytest.mark.parametrize("hf_data", [{}, {"text": ""}, {"audio": None}, {"audio": []}]) +def test_hf_processor_without_audio(asr_processor, hf_data): + """The media hook must not require audio or call HF for absent waveforms.""" + processor, _ = asr_processor + assert not processor._call_hf_processor(hf_data, {}) + + +@pytest.mark.parametrize("hf_data", [{}, {"text": ""}]) +def test_postprocessing_without_audio_features(asr_processor, hf_data): + """Non-audio output must pass through even when HF input is nonempty.""" + processor, _ = asr_processor + processed = BatchFeature({"input_ids": torch.tensor([[1, 2]])}) + actual = processor._postprocess_hf_mm_data(hf_data, {}, processed) + assert actual is processed + assert set(actual) == {"input_ids"} + torch.testing.assert_close(actual["input_ids"], torch.tensor([[1, 2]])) + + +def test_chat_template_processing(asr_processor): + """The generic processor loader must still render audio chat prompts.""" + processor, native = asr_processor + chat_processor = cached_processor_from_config(processor.info.ctx.model_config) + messages = [ + {"role": "system", "content": ""}, + {"role": "user", "content": [{"type": "audio", "audio": "unused"}]}, + ] + prompt = chat_processor.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True + ) + expected_prompt = native.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True + ) + assert prompt == expected_prompt + audio = np.random.RandomState(42).randn(16001).astype(np.float32) + expected = native(text=expected_prompt, audio=audio, return_tensors="pt") + actual = processor( + prompt, + mm_items=processor.info.parse_mm_data({"audio": audio}), + hf_processor_mm_kwargs={}, + ) + assert actual["prompt_token_ids"] == expected["input_ids"][0].tolist() + + +def test_precomputed_audio_features(asr_processor): + """Precomputed features must bypass waveform extraction unchanged.""" + processor, _ = asr_processor + processor.info.ctx.get_mm_config().enable_mm_embeds = True + audio = np.random.RandomState(42).randn(16001).astype(np.float32) + expected = processor( + _AUDIO_PROMPT, + mm_items=processor.info.parse_mm_data({"audio": audio}), + hf_processor_mm_kwargs={}, + ) + features = expected["mm_kwargs"].get_data() + actual = processor( + _AUDIO_PROMPT, + mm_items=processor.info.parse_mm_data({"audio": features}), + hf_processor_mm_kwargs={}, + ) + assert actual["prompt_token_ids"] == expected["prompt_token_ids"] + assert actual["mm_placeholders"] == expected["mm_placeholders"] + assert batched_tensors_equal(actual["mm_kwargs"].get_data(), features) diff --git a/vllm/model_executor/models/qwen3_asr.py b/vllm/model_executor/models/qwen3_asr.py index c8a0849fc5..161d1efacd 100644 --- a/vllm/model_executor/models/qwen3_asr.py +++ b/vllm/model_executor/models/qwen3_asr.py @@ -29,6 +29,7 @@ import regex as re import torch import torch.nn as nn from transformers.feature_extraction_utils import BatchFeature +from transformers.models.qwen3_asr import Qwen3ASRFeatureExtractor, Qwen3ASRProcessor from transformers.models.whisper import WhisperFeatureExtractor from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig @@ -54,7 +55,6 @@ from vllm.model_executor.models.qwen2_5_omni_thinker import ( from vllm.model_executor.models.qwen3 import Qwen3ForCausalLM from vllm.model_executor.models.qwen3_omni_moe_thinker import ( Qwen3OmniMoeAudioEncoder, - Qwen3OmniMoeThinkerMultiModalProcessor, ) from vllm.model_executor.models.utils import ( AutoWeightsLoader, @@ -80,6 +80,7 @@ from vllm.multimodal.parse import ( ) from vllm.multimodal.processing import ( BaseDummyInputsBuilder, + BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, @@ -92,9 +93,6 @@ from vllm.transformers_utils.configs.qwen3_asr import ( Qwen3ASRThinkerConfig, ) from vllm.transformers_utils.processor import cached_processor_from_config -from vllm.transformers_utils.processors.qwen3_asr import ( - Qwen3ASRProcessor, -) from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.utils.torch_utils import async_tensor_h2d @@ -188,18 +186,19 @@ class Qwen3ASRProcessingInfo(BaseProcessingInfo): return self.ctx.get_hf_config(Qwen3ASRConfig).thinker_config def get_hf_processor(self, **kwargs: object) -> Qwen3ASRProcessor: - processor = self.ctx.get_hf_processor( + return self.ctx.get_hf_processor( Qwen3ASRProcessor, **kwargs, ) - if not hasattr(processor, "audio_token"): - processor.audio_token = "<|audio_pad|>" - return processor - def get_feature_extractor(self, **kwargs: object) -> WhisperFeatureExtractor: + def get_feature_extractor( + self, **kwargs: object + ) -> WhisperFeatureExtractor | Qwen3ASRFeatureExtractor: hf_processor = self.get_hf_processor(**kwargs) feature_extractor = hf_processor.feature_extractor - assert isinstance(feature_extractor, WhisperFeatureExtractor) + assert isinstance( + feature_extractor, (WhisperFeatureExtractor, Qwen3ASRFeatureExtractor) + ) return feature_extractor def get_supported_mm_limits(self) -> Mapping[str, int | None]: @@ -275,8 +274,60 @@ class Qwen3ASRMultiModalDataParser(MultiModalDataParser): class Qwen3ASRMultiModalProcessor( - Qwen3OmniMoeThinkerMultiModalProcessor[Qwen3ASRProcessingInfo] + BaseMultiModalProcessor[Qwen3ASRProcessingInfo], ): + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + + def _call_hf_processor( + self, + hf_data: Mapping[str, object], + hf_kwargs: Mapping[str, object], + ) -> BatchFeature: + audios = hf_data.get("audio") + if not audios: + return BatchFeature() + assert isinstance(audios, list) + + if len(audios) == 1: + return super()._call_hf_processor(hf_data, hf_kwargs) + + # HF requires one text per audio, and batch padding changes clip features. + # Keep clips independent so partial cache hits cannot change their inputs. + text = self._get_hf_mm_text({"audio": 1}) + outputs: dict[str, list[torch.Tensor]] = { + "input_features": [], + "input_features_mask": [], + } + for audio in audios: + processed = super()._call_hf_processor( + dict(hf_data, text=text, audio=audio), hf_kwargs + ) + for key, values in outputs.items(): + values.append(processed[key][0]) + + return BatchFeature(outputs) + + def _postprocess_hf_mm_data( + self, + hf_data: Mapping[str, object], + hf_kwargs: Mapping[str, object], + processed_data: BatchFeature, + ) -> BatchFeature: + if "input_features" not in processed_data: + return processed_data + + features = processed_data.pop("input_features") + masks = processed_data.pop("input_features_mask") + processed_data["input_audio_features"] = torch.cat( + [feature[:, mask.bool()] for feature, mask in zip(features, masks)], dim=1 + ) + processed_data["feature_attention_mask"] = masks + processed_data["audio_feature_lengths"] = torch.stack( + [mask.sum() for mask in masks] + ) + return processed_data + def _get_mm_fields_config( self, hf_inputs: BatchFeature, diff --git a/vllm/transformers_utils/processors/__init__.py b/vllm/transformers_utils/processors/__init__.py index f2af4901e2..db45e49d5a 100644 --- a/vllm/transformers_utils/processors/__init__.py +++ b/vllm/transformers_utils/processors/__init__.py @@ -38,7 +38,6 @@ __all__ = [ "OpenVLAProcessor", "OvisProcessor", "Ovis2_5Processor", - "Qwen3ASRProcessor", "Step3VLProcessor", "InklingProcessor", "InklingImageProcessor", @@ -74,7 +73,6 @@ _CLASS_TO_MODULE: dict[str, str] = { "OpenVLAProcessor": "vllm.transformers_utils.processors.openvla", "OvisProcessor": "vllm.transformers_utils.processors.ovis", "Ovis2_5Processor": "vllm.transformers_utils.processors.ovis2_5", - "Qwen3ASRProcessor": "vllm.transformers_utils.processors.qwen3_asr", "Step3VLProcessor": "vllm.transformers_utils.processors.step3_vl", "InklingProcessor": "vllm.transformers_utils.processors.inkling", "InklingImageProcessor": "vllm.transformers_utils.processors.inkling", diff --git a/vllm/transformers_utils/processors/qwen3_asr.py b/vllm/transformers_utils/processors/qwen3_asr.py deleted file mode 100644 index 55d3853792..0000000000 --- a/vllm/transformers_utils/processors/qwen3_asr.py +++ /dev/null @@ -1,229 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# ruff: noqa -# mypy: ignore-errors -# coding=utf-8 -# Copyright 2026 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import regex as re - -import numpy as np - -from transformers import AutoProcessor -from transformers.audio_utils import AudioInput -from transformers.feature_extraction_utils import BatchFeature -from transformers.processing_utils import ProcessingKwargs, ProcessorMixin -from transformers.tokenization_utils_base import TextInput - - -class Qwen3ASRProcessorKwargs(ProcessingKwargs, total=False): - _defaults = { - "text_kwargs": { - "padding": False, - "padding_side": "left", - }, - "audio_kwargs": { - "sampling_rate": 16000, - "padding": True, - "return_attention_mask": True, - }, - } - - -def _get_feat_extract_output_lengths(input_lengths): - """ - Computes the output length of the convolutional layers and the output length of the audio encoder - """ - - input_lengths_leave = input_lengths % 100 - feat_lengths = (input_lengths_leave - 1) // 2 + 1 - output_lengths = ( - ((feat_lengths - 1) // 2 + 1 - 1) // 2 + 1 + (input_lengths // 100) * 13 - ) - return output_lengths - - -class Qwen3ASRProcessor(ProcessorMixin): - r""" - Constructs a Qwen3ASR processor. - [`Qwen3ASRProcessor`] offers all the functionalities of [`WhisperFeatureExtractor`], and [`Qwen2TokenizerFast`]. See the - [`~Qwen3ASRProcessor.__call__`] and [`~Qwen3ASRProcessor.decode`] for more information. - - Args: - feature_extractor ([`WhisperFeatureExtractor`], *optional*): - The audio feature extractor. - tokenizer ([`Qwen2TokenizerFast`], *optional*): - The text tokenizer. - chat_template (`Optional[str]`, *optional*): - The Jinja template to use for formatting the conversation. If not provided, the default chat template is used. - """ - - attributes = ["feature_extractor", "tokenizer"] - feature_extractor_class = "WhisperFeatureExtractor" - tokenizer_class = ("Qwen2Tokenizer", "Qwen2TokenizerFast") - - def __init__(self, feature_extractor=None, tokenizer=None, chat_template=None): - super().__init__(feature_extractor, tokenizer, chat_template=chat_template) - self.audio_token = self.tokenizer.audio_token - self.audio_bos_token = self.tokenizer.audio_bos_token - self.audio_eos_token = self.tokenizer.audio_eos_token - - def __call__( - self, - text: TextInput = None, - audio: AudioInput = None, - **kwargs, - ) -> BatchFeature: - """ - Main method to prepare for the model one or several sequences(s) and audio(s). This method forwards the `text` - and `kwargs` arguments to Qwen2TokenizerFast's [`~Qwen2TokenizerFast.__call__`] if `text` is not `None` to encode - the text. To prepare the audio(s), this method forwards the `audio` and `kwargs` arguments to - WhisperFeatureExtractor's [`~WhisperFeatureExtractor.__call__`] if `audio` is not `None`. Please refer to the doctsring - of the above two methods for more information. - - Args: - text (`str`, `List[str]`, `List[List[str]]`): - The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings - (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set - `is_split_into_words=True` (to lift the ambiguity with a batch of sequences). - audio (`np.ndarray`, `List[np.ndarray]`): - The audio or batch of audio to be prepared. Each audio can be a NumPy array. - """ - - if text is None: - raise ValueError("You need to specify either a `text` input to process.") - - output_kwargs = self._merge_kwargs( - Qwen3ASRProcessorKwargs, - tokenizer_init_kwargs=self.tokenizer.init_kwargs, - **kwargs, - ) - - if audio is not None: - output_kwargs["audio_kwargs"]["padding"] = True - output_kwargs["audio_kwargs"]["truncation"] = False - audio_inputs = self.feature_extractor( - audio, **output_kwargs["audio_kwargs"] - ) - audio_inputs["feature_attention_mask"] = audio_inputs.pop( - "attention_mask" - ) # rename feature_attention_mask to prevent conflicts later on - audio_inputs["input_features"] = audio_inputs.pop( - "input_features" - ) # rename input_features to prevent conflicts later on - audio_lengths = iter( - _get_feat_extract_output_lengths( - audio_inputs["feature_attention_mask"].sum(-1) - ) - ) - else: - audio_inputs = {} - audio_lengths = iter([]) - - if not isinstance(text, list): - text = [text] - - text = self.replace_multimodal_special_tokens( - text, - audio_lengths, - ) - - texts_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"]) - - return BatchFeature( - data={**texts_inputs, **audio_inputs}, - tensor_type=kwargs.get("return_tensors"), - ) - - def replace_multimodal_special_tokens( - self, - text, - audio_lengths, - ): - processed_text = [] - for sample in text: - positions = [] - special_tokens = [re.escape(tok) for tok in [self.audio_token]] - pattern = "|".join(special_tokens) - positions = sorted( - [ - (match.start(), match.group()) - for match in re.finditer(pattern, sample) - ] - ) - positions.sort(key=lambda x: x[0]) - - for _, special_token in positions: - if special_token == self.audio_token: - sample = sample.replace( - self.audio_token, - "<|audio_placeholder|>" * next(audio_lengths), - 1, - ) - - sample = sample.replace("<|audio_placeholder|>", self.audio_token) - processed_text.append(sample) - return processed_text - - def get_chunked_index( - self, token_indices: np.ndarray, tokens_per_chunk: int - ) -> list[tuple[int, int]]: - """ - Splits token index list into chunks based on token value ranges. - - Given a list of token indices, returns a list of (start, end) index tuples representing - slices of the list where the token values fall within successive ranges of `tokens_per_chunk`. - - For example, if `tokens_per_chunk` is 1000, the function will create chunks such that: - - the first chunk contains token values < 1000, - - the second chunk contains values >= 1000 and < 2000, and so on. - - Parameters: - token_indices (`np.ndarray`): A monotonically increasing list of token index values. - tokens_per_chunk (`int`): Number of tokens per chunk (used as the chunk size threshold). - - Returns: - `list[tuple[int, int]]`: A list of tuples, each representing the start (inclusive) - and end (exclusive) indices of a chunk in `token_indices`. - """ - - def _iter(): - i, start_idx = 0, 0 # skip bos token - current_chunk = 1 - while i < len(token_indices): # skip eos token - if token_indices[i] >= current_chunk * tokens_per_chunk: - yield (start_idx, i) - start_idx = i - current_chunk += 1 - i += 1 - yield (start_idx, len(token_indices)) - - return list(_iter()) - - def apply_chat_template(self, conversations, chat_template=None, **kwargs): - kwargs["return_dict"] = False - return super().apply_chat_template(conversations, chat_template, **kwargs) - - @property - def model_input_names(self): - tokenizer_input_names = self.tokenizer.model_input_names - feature_extractor_input_names = self.feature_extractor.model_input_names - return list( - dict.fromkeys( - tokenizer_input_names - + feature_extractor_input_names - + ["feature_attention_mask"] - ) - )