diff --git a/tests/transformers_utils/processors/__init__.py b/tests/transformers_utils/processors/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/tests/transformers_utils/processors/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/tests/transformers_utils/processors/test_voxtral.py b/tests/transformers_utils/processors/test_voxtral.py new file mode 100644 index 0000000000..0ca8f1a94d --- /dev/null +++ b/tests/transformers_utils/processors/test_voxtral.py @@ -0,0 +1,127 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for ``MistralCommonFeatureExtractor.fetch_audio``. + +``transformers>=5.10`` adds a ``ProcessorMixin.prepare_inputs_layout`` helper +that calls ``self.feature_extractor.fetch_audio(...)`` unconditionally. The +duck-typed :class:`MistralCommonFeatureExtractor` previously did not implement +that method, so loading any voxtral model under transformers 5.10.x raised +``AttributeError: 'MistralCommonFeatureExtractor' object has no attribute +'fetch_audio'``. These tests pin the new ``fetch_audio`` method to the same +contract as ``transformers.SequenceFeatureExtractor.fetch_audio``. +""" + +import numpy as np +import pytest +import torch + +from vllm.tokenizers.mistral import MistralTokenizer +from vllm.transformers_utils.processors.voxtral import ( + MistralCommonFeatureExtractor, +) + + +@pytest.fixture(scope="module") +def feature_extractor() -> MistralCommonFeatureExtractor: + tokenizer = MistralTokenizer.from_pretrained("mistralai/Voxtral-Mini-3B-2507") + return MistralCommonFeatureExtractor(tokenizer.instruct.audio_encoder) + + +@pytest.mark.parametrize( + "audio", + [ + np.zeros(1024, dtype=np.float32), + torch.zeros(1024), + [0.0, 1.0, 2.0], + ], + ids=["numpy_array", "torch_tensor", "list_of_floats"], +) +def test_fetch_audio_passes_through( + feature_extractor: MistralCommonFeatureExtractor, audio +): + result = feature_extractor.fetch_audio(audio) + assert result is audio + + +def test_fetch_audio_recurses_over_list_of_arrays( + feature_extractor: MistralCommonFeatureExtractor, +): + a = np.zeros(8, dtype=np.float32) + b = np.ones(8, dtype=np.float32) + result = feature_extractor.fetch_audio([a, b]) + assert isinstance(result, list) + assert len(result) == 2 + assert result[0] is a + assert result[1] is b + + +def test_fetch_audio_uses_self_sampling_rate_when_none( + monkeypatch, feature_extractor: MistralCommonFeatureExtractor +): + """If ``sampling_rate`` is None, ``self.sampling_rate`` must be used. + + Verified indirectly via the recursion path: when we pass a list of arrays + without sampling_rate, recursive calls receive the resolved rate. + """ + captured: list[int | None] = [] + original = feature_extractor.fetch_audio + + def spy(audio, sampling_rate=None): + captured.append(sampling_rate) + return original(audio, sampling_rate=sampling_rate) + + monkeypatch.setattr(feature_extractor, "fetch_audio", spy) + feature_extractor.fetch_audio([np.zeros(4, dtype=np.float32)]) + # Top-level call has sampling_rate=None; inner recursive call sees the + # resolved rate from self.sampling_rate. + assert captured[0] is None + assert captured[1] == 16000 + + +def test_fetch_audio_explicit_sampling_rate_propagates( + monkeypatch, feature_extractor: MistralCommonFeatureExtractor +): + captured: list[int | None] = [] + original = feature_extractor.fetch_audio + + def spy(audio, sampling_rate=None): + captured.append(sampling_rate) + return original(audio, sampling_rate=sampling_rate) + + monkeypatch.setattr(feature_extractor, "fetch_audio", spy) + feature_extractor.fetch_audio([np.zeros(4, dtype=np.float32)], sampling_rate=8000) + assert captured[0] == 8000 + assert captured[1] == 8000 + + +def test_fetch_audio_rejects_unsupported_type( + feature_extractor: MistralCommonFeatureExtractor, +): + with pytest.raises(TypeError, match="only a numpy array"): + feature_extractor.fetch_audio(42) # type: ignore[arg-type] + + +def test_fetch_audio_str_delegates_to_load_audio( + monkeypatch, feature_extractor: MistralCommonFeatureExtractor +): + """A str input must round-trip through ``transformers.audio_utils.load_audio``. + + We monkey-patch ``load_audio`` so the test stays offline (no real URL/path + fetched) and still asserts the delegation contract. + """ + sentinel = np.array([0.5, -0.5], dtype=np.float32) + received: dict[str, object] = {} + + def fake_load_audio(path, sampling_rate=None): + received["path"] = path + received["sampling_rate"] = sampling_rate + return sentinel + + import transformers.audio_utils + + monkeypatch.setattr(transformers.audio_utils, "load_audio", fake_load_audio) + + result = feature_extractor.fetch_audio("/tmp/fake.wav") + assert result is sentinel + assert received["path"] == "/tmp/fake.wav" + assert received["sampling_rate"] == 16000 diff --git a/vllm/transformers_utils/processors/voxtral.py b/vllm/transformers_utils/processors/voxtral.py index 829bab2d41..3abe660611 100644 --- a/vllm/transformers_utils/processors/voxtral.py +++ b/vllm/transformers_utils/processors/voxtral.py @@ -53,6 +53,54 @@ class MistralCommonFeatureExtractor: def get_num_audio_tokens(self, audio_length: int) -> int: return ceil(audio_length / (self.sampling_rate // self.frame_rate)) + def fetch_audio(self, audio_url_or_urls, sampling_rate=None): + """HF-compatible duck-typed ``fetch_audio``. + + Mirrors :meth:`transformers.SequenceFeatureExtractor.fetch_audio` so + :class:`transformers.ProcessorMixin.prepare_inputs_layout` (added in + transformers 5.10) works on this duck-typed feature extractor. Older + transformers versions never invoke this method, so the addition is a + no-op there. + + Accepts the same shapes as ``SequenceFeatureExtractor.fetch_audio``: + + * ``np.ndarray`` / ``torch.Tensor`` — returned as-is. + * ``list[float]`` — returned as-is (a single audio sample). + * ``str`` URL or path — delegated to + :func:`transformers.audio_utils.load_audio`. + * ``list`` of any of the above — recursed element-wise. + + ``ProcessorMixin.prepare_inputs_layout`` always passes already-decoded + audio (numpy array or torch tensor), so the str / list-of-str branches + exist only to keep the contract identical to the upstream method. + + The semantics of ``transformers.audio_utils.is_valid_audio`` differ + between transformers versions (5.9 only accepts ndarray/tensor; 5.10 + also accepts ``list[float]``). We detect ``list[float]`` explicitly to + keep behavior identical across versions. + """ + from transformers.audio_utils import is_valid_audio + + sampling_rate = sampling_rate if sampling_rate else self.sampling_rate + if is_valid_audio(audio_url_or_urls): + return audio_url_or_urls + if isinstance(audio_url_or_urls, (list, tuple)): + if audio_url_or_urls and isinstance(audio_url_or_urls[0], float): + # A single audio represented as ``list[float]``. + return audio_url_or_urls + return [ + self.fetch_audio(x, sampling_rate=sampling_rate) + for x in audio_url_or_urls + ] + if isinstance(audio_url_or_urls, str): + from transformers.audio_utils import load_audio + + return load_audio(audio_url_or_urls, sampling_rate=sampling_rate) + raise TypeError( + "only a numpy array, torch tensor, str URL/path, or list of those " + f"is supported but got type={type(audio_url_or_urls)}" + ) + class MistralCommonVoxtralProcessor(ProcessorMixin): attributes = ["feature_extractor", "tokenizer"]