[Bugfix][Voxtral] Add fetch_audio to MistralCommonFeatureExtractor (transformers>=5.10 compat) (#44559)
Signed-off-by: Yadan Wei <weiyadan@amazon.com> Co-authored-by: Yadan Wei <weiyadan@amazon.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Yadan Wei
Claude
parent
f87df1df9e
commit
c9b4b184b4
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,127 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for ``MistralCommonFeatureExtractor.fetch_audio``.
|
||||
|
||||
``transformers>=5.10`` adds a ``ProcessorMixin.prepare_inputs_layout`` helper
|
||||
that calls ``self.feature_extractor.fetch_audio(...)`` unconditionally. The
|
||||
duck-typed :class:`MistralCommonFeatureExtractor` previously did not implement
|
||||
that method, so loading any voxtral model under transformers 5.10.x raised
|
||||
``AttributeError: 'MistralCommonFeatureExtractor' object has no attribute
|
||||
'fetch_audio'``. These tests pin the new ``fetch_audio`` method to the same
|
||||
contract as ``transformers.SequenceFeatureExtractor.fetch_audio``.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
from vllm.transformers_utils.processors.voxtral import (
|
||||
MistralCommonFeatureExtractor,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def feature_extractor() -> MistralCommonFeatureExtractor:
|
||||
tokenizer = MistralTokenizer.from_pretrained("mistralai/Voxtral-Mini-3B-2507")
|
||||
return MistralCommonFeatureExtractor(tokenizer.instruct.audio_encoder)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"audio",
|
||||
[
|
||||
np.zeros(1024, dtype=np.float32),
|
||||
torch.zeros(1024),
|
||||
[0.0, 1.0, 2.0],
|
||||
],
|
||||
ids=["numpy_array", "torch_tensor", "list_of_floats"],
|
||||
)
|
||||
def test_fetch_audio_passes_through(
|
||||
feature_extractor: MistralCommonFeatureExtractor, audio
|
||||
):
|
||||
result = feature_extractor.fetch_audio(audio)
|
||||
assert result is audio
|
||||
|
||||
|
||||
def test_fetch_audio_recurses_over_list_of_arrays(
|
||||
feature_extractor: MistralCommonFeatureExtractor,
|
||||
):
|
||||
a = np.zeros(8, dtype=np.float32)
|
||||
b = np.ones(8, dtype=np.float32)
|
||||
result = feature_extractor.fetch_audio([a, b])
|
||||
assert isinstance(result, list)
|
||||
assert len(result) == 2
|
||||
assert result[0] is a
|
||||
assert result[1] is b
|
||||
|
||||
|
||||
def test_fetch_audio_uses_self_sampling_rate_when_none(
|
||||
monkeypatch, feature_extractor: MistralCommonFeatureExtractor
|
||||
):
|
||||
"""If ``sampling_rate`` is None, ``self.sampling_rate`` must be used.
|
||||
|
||||
Verified indirectly via the recursion path: when we pass a list of arrays
|
||||
without sampling_rate, recursive calls receive the resolved rate.
|
||||
"""
|
||||
captured: list[int | None] = []
|
||||
original = feature_extractor.fetch_audio
|
||||
|
||||
def spy(audio, sampling_rate=None):
|
||||
captured.append(sampling_rate)
|
||||
return original(audio, sampling_rate=sampling_rate)
|
||||
|
||||
monkeypatch.setattr(feature_extractor, "fetch_audio", spy)
|
||||
feature_extractor.fetch_audio([np.zeros(4, dtype=np.float32)])
|
||||
# Top-level call has sampling_rate=None; inner recursive call sees the
|
||||
# resolved rate from self.sampling_rate.
|
||||
assert captured[0] is None
|
||||
assert captured[1] == 16000
|
||||
|
||||
|
||||
def test_fetch_audio_explicit_sampling_rate_propagates(
|
||||
monkeypatch, feature_extractor: MistralCommonFeatureExtractor
|
||||
):
|
||||
captured: list[int | None] = []
|
||||
original = feature_extractor.fetch_audio
|
||||
|
||||
def spy(audio, sampling_rate=None):
|
||||
captured.append(sampling_rate)
|
||||
return original(audio, sampling_rate=sampling_rate)
|
||||
|
||||
monkeypatch.setattr(feature_extractor, "fetch_audio", spy)
|
||||
feature_extractor.fetch_audio([np.zeros(4, dtype=np.float32)], sampling_rate=8000)
|
||||
assert captured[0] == 8000
|
||||
assert captured[1] == 8000
|
||||
|
||||
|
||||
def test_fetch_audio_rejects_unsupported_type(
|
||||
feature_extractor: MistralCommonFeatureExtractor,
|
||||
):
|
||||
with pytest.raises(TypeError, match="only a numpy array"):
|
||||
feature_extractor.fetch_audio(42) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_fetch_audio_str_delegates_to_load_audio(
|
||||
monkeypatch, feature_extractor: MistralCommonFeatureExtractor
|
||||
):
|
||||
"""A str input must round-trip through ``transformers.audio_utils.load_audio``.
|
||||
|
||||
We monkey-patch ``load_audio`` so the test stays offline (no real URL/path
|
||||
fetched) and still asserts the delegation contract.
|
||||
"""
|
||||
sentinel = np.array([0.5, -0.5], dtype=np.float32)
|
||||
received: dict[str, object] = {}
|
||||
|
||||
def fake_load_audio(path, sampling_rate=None):
|
||||
received["path"] = path
|
||||
received["sampling_rate"] = sampling_rate
|
||||
return sentinel
|
||||
|
||||
import transformers.audio_utils
|
||||
|
||||
monkeypatch.setattr(transformers.audio_utils, "load_audio", fake_load_audio)
|
||||
|
||||
result = feature_extractor.fetch_audio("/tmp/fake.wav")
|
||||
assert result is sentinel
|
||||
assert received["path"] == "/tmp/fake.wav"
|
||||
assert received["sampling_rate"] == 16000
|
||||
@@ -53,6 +53,54 @@ class MistralCommonFeatureExtractor:
|
||||
def get_num_audio_tokens(self, audio_length: int) -> int:
|
||||
return ceil(audio_length / (self.sampling_rate // self.frame_rate))
|
||||
|
||||
def fetch_audio(self, audio_url_or_urls, sampling_rate=None):
|
||||
"""HF-compatible duck-typed ``fetch_audio``.
|
||||
|
||||
Mirrors :meth:`transformers.SequenceFeatureExtractor.fetch_audio` so
|
||||
:class:`transformers.ProcessorMixin.prepare_inputs_layout` (added in
|
||||
transformers 5.10) works on this duck-typed feature extractor. Older
|
||||
transformers versions never invoke this method, so the addition is a
|
||||
no-op there.
|
||||
|
||||
Accepts the same shapes as ``SequenceFeatureExtractor.fetch_audio``:
|
||||
|
||||
* ``np.ndarray`` / ``torch.Tensor`` — returned as-is.
|
||||
* ``list[float]`` — returned as-is (a single audio sample).
|
||||
* ``str`` URL or path — delegated to
|
||||
:func:`transformers.audio_utils.load_audio`.
|
||||
* ``list`` of any of the above — recursed element-wise.
|
||||
|
||||
``ProcessorMixin.prepare_inputs_layout`` always passes already-decoded
|
||||
audio (numpy array or torch tensor), so the str / list-of-str branches
|
||||
exist only to keep the contract identical to the upstream method.
|
||||
|
||||
The semantics of ``transformers.audio_utils.is_valid_audio`` differ
|
||||
between transformers versions (5.9 only accepts ndarray/tensor; 5.10
|
||||
also accepts ``list[float]``). We detect ``list[float]`` explicitly to
|
||||
keep behavior identical across versions.
|
||||
"""
|
||||
from transformers.audio_utils import is_valid_audio
|
||||
|
||||
sampling_rate = sampling_rate if sampling_rate else self.sampling_rate
|
||||
if is_valid_audio(audio_url_or_urls):
|
||||
return audio_url_or_urls
|
||||
if isinstance(audio_url_or_urls, (list, tuple)):
|
||||
if audio_url_or_urls and isinstance(audio_url_or_urls[0], float):
|
||||
# A single audio represented as ``list[float]``.
|
||||
return audio_url_or_urls
|
||||
return [
|
||||
self.fetch_audio(x, sampling_rate=sampling_rate)
|
||||
for x in audio_url_or_urls
|
||||
]
|
||||
if isinstance(audio_url_or_urls, str):
|
||||
from transformers.audio_utils import load_audio
|
||||
|
||||
return load_audio(audio_url_or_urls, sampling_rate=sampling_rate)
|
||||
raise TypeError(
|
||||
"only a numpy array, torch tensor, str URL/path, or list of those "
|
||||
f"is supported but got type={type(audio_url_or_urls)}"
|
||||
)
|
||||
|
||||
|
||||
class MistralCommonVoxtralProcessor(ProcessorMixin):
|
||||
attributes = ["feature_extractor", "tokenizer"]
|
||||
|
||||
Reference in New Issue
Block a user