From 3d3b96488f5f7d94b1ab63919e0f1e7922a9ded6 Mon Sep 17 00:00:00 2001 From: Julien Denize <40604584+juliendenize@users.noreply.github.com> Date: Fri, 26 Jun 2026 20:06:31 +0200 Subject: [PATCH] Migrate Voxtral to mistral-common 1.11.5 audio API (#46705) Signed-off-by: Julien Denize <40604584+juliendenize@users.noreply.github.com> --- examples/generate/multimodal/audio_language_offline.py | 7 ++----- requirements/common.txt | 2 +- requirements/test/cuda.in | 2 +- requirements/test/cuda.txt | 2 +- requirements/test/nightly-torch.txt | 2 +- requirements/test/rocm.in | 2 +- requirements/test/rocm.txt | 2 +- requirements/test/xpu.txt | 2 +- tests/models/multimodal/generation/test_voxtral.py | 8 +++----- .../multimodal/generation/test_voxtral_realtime.py | 7 +++---- vllm/model_executor/models/voxtral.py | 9 +++++---- vllm/model_executor/models/voxtral_realtime.py | 6 ++---- 12 files changed, 22 insertions(+), 29 deletions(-) diff --git a/examples/generate/multimodal/audio_language_offline.py b/examples/generate/multimodal/audio_language_offline.py index c480f1b414..12a38cf41c 100644 --- a/examples/generate/multimodal/audio_language_offline.py +++ b/examples/generate/multimodal/audio_language_offline.py @@ -463,16 +463,15 @@ def run_ultravox(question: str, audio_count: int) -> ModelRequestData: # Voxtral # Make sure to install mistral-common[audio]. def run_voxtral(question: str, audio_count: int) -> ModelRequestData: - from mistral_common.audio import Audio from mistral_common.protocol.instruct.chunk import ( AudioChunk, - RawAudio, TextChunk, ) from mistral_common.protocol.instruct.messages import ( UserMessage, ) from mistral_common.protocol.instruct.request import ChatCompletionRequest + from mistral_common.tokens.tokenizers.audio import Audio from mistral_common.tokens.tokenizers.mistral import MistralTokenizer model_name = "mistralai/Voxtral-Mini-3B-2507" @@ -495,9 +494,7 @@ def run_voxtral(question: str, audio_count: int) -> ModelRequestData: Audio.from_file(str(audio_assets[i].get_local_path()), strict=False) for i in range(audio_count) ] - audio_chunks = [ - AudioChunk(input_audio=RawAudio.from_audio(audio)) for audio in audios - ] + audio_chunks = [AudioChunk.from_audio(audio) for audio in audios] messages = [UserMessage(content=[*audio_chunks, text_chunk])] diff --git a/requirements/common.txt b/requirements/common.txt index a5d74e14e6..1652480c22 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -33,7 +33,7 @@ partial-json-parser # used for parsing partial JSON outputs jsonschema >= 4.23.0 # required for MiniMax M3 tool schema validation pyzmq >= 25.0.0 msgspec -mistral_common[image] >= 1.11.3 +mistral_common[image] >= 1.11.5 opencv-python-headless >= 4.13.0 # required for video IO pyyaml six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12 diff --git a/requirements/test/cuda.in b/requirements/test/cuda.in index a7fc65def8..03218c75e1 100644 --- a/requirements/test/cuda.in +++ b/requirements/test/cuda.in @@ -31,7 +31,7 @@ torchaudio==2.11.0 torchvision==0.26.0 transformers_stream_generator # required for qwen-vl test matplotlib # required for qwen-vl test -mistral_common[image,audio] >= 1.11.3 # required for voxtral test +mistral_common[image,audio] >= 1.11.5 # required for voxtral test num2words # required for smolvlm test open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_common.py opencv-python-headless >= 4.13.0 # required for video test diff --git a/requirements/test/cuda.txt b/requirements/test/cuda.txt index 76c343b91b..1a9fe6f16a 100644 --- a/requirements/test/cuda.txt +++ b/requirements/test/cuda.txt @@ -398,7 +398,7 @@ mbstrdecoder==1.1.3 # typepy mdurl==0.1.2 # via markdown-it-py -mistral-common==1.11.3 +mistral-common==1.11.5 # via # -c requirements/common.txt # -r requirements/test/cuda.in diff --git a/requirements/test/nightly-torch.txt b/requirements/test/nightly-torch.txt index a58e0fa248..08f721771c 100644 --- a/requirements/test/nightly-torch.txt +++ b/requirements/test/nightly-torch.txt @@ -23,7 +23,7 @@ jiwer # required for audio tests timm # required for internvl test transformers_stream_generator # required for qwen-vl test matplotlib # required for qwen-vl test -mistral_common[image,audio] >= 1.11.3 # required for voxtral test +mistral_common[image,audio] >= 1.11.5 # required for voxtral test num2words # required for smolvlm test opencv-python-headless >= 4.13.0 # required for video test datamodel_code_generator # required for minicpm3 test diff --git a/requirements/test/rocm.in b/requirements/test/rocm.in index 046ca09ff7..6a38f384f1 100644 --- a/requirements/test/rocm.in +++ b/requirements/test/rocm.in @@ -30,7 +30,7 @@ tblib # for pickling test exceptions timm>=1.0.17 # required for internvl and gemma3n-mm test transformers_stream_generator # required for qwen-vl test matplotlib # required for qwen-vl test -mistral_common[image,audio]>=1.11.3 # required for voxtral test +mistral_common[image,audio]>=1.11.5 # required for voxtral test num2words # required for smolvlm test open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_common.py opencv-python-headless>=4.13.0 # required for video test diff --git a/requirements/test/rocm.txt b/requirements/test/rocm.txt index 842d2ff318..726aad9a67 100644 --- a/requirements/test/rocm.txt +++ b/requirements/test/rocm.txt @@ -499,7 +499,7 @@ mcp==1.27.0 # via -r requirements/test/../common.txt mdurl==0.1.2 # via markdown-it-py -mistral-common==1.11.3 +mistral-common==1.11.5 # via # -c requirements/common.txt # -r requirements/test/../common.txt diff --git a/requirements/test/xpu.txt b/requirements/test/xpu.txt index 40f23b95d1..2b938e3b58 100644 --- a/requirements/test/xpu.txt +++ b/requirements/test/xpu.txt @@ -264,7 +264,7 @@ mbstrdecoder==1.1.4 # typepy mdurl==0.1.2 # via markdown-it-py -mistral-common==1.11.3 +mistral-common==1.11.5 # via # -c requirements/common.txt # -r requirements/test/xpu.in diff --git a/tests/models/multimodal/generation/test_voxtral.py b/tests/models/multimodal/generation/test_voxtral.py index 82db1dc681..a6e8f3ff18 100644 --- a/tests/models/multimodal/generation/test_voxtral.py +++ b/tests/models/multimodal/generation/test_voxtral.py @@ -4,9 +4,9 @@ import json import pytest -from mistral_common.audio import Audio -from mistral_common.protocol.instruct.chunk import AudioChunk, RawAudio, TextChunk +from mistral_common.protocol.instruct.chunk import AudioChunk, TextChunk from mistral_common.protocol.instruct.messages import UserMessage +from mistral_common.tokens.tokenizers.audio import Audio from transformers import VoxtralForConditionalGeneration from vllm.tokenizers.mistral import MistralTokenizer @@ -36,9 +36,7 @@ def _get_prompt(audio_assets: AudioTestAssets, question: str) -> list[int]: Audio.from_file(str(asset.get_local_path()), strict=False) for asset in audio_assets ] - audio_chunks = [ - AudioChunk(input_audio=RawAudio.from_audio(audio)) for audio in audios - ] + audio_chunks = [AudioChunk.from_audio(audio) for audio in audios] messages = [ UserMessage(content=[*audio_chunks, TextChunk(text=question)]).to_openai() diff --git a/tests/models/multimodal/generation/test_voxtral_realtime.py b/tests/models/multimodal/generation/test_voxtral_realtime.py index ca43e7b51f..be677ccb57 100644 --- a/tests/models/multimodal/generation/test_voxtral_realtime.py +++ b/tests/models/multimodal/generation/test_voxtral_realtime.py @@ -4,12 +4,11 @@ import contextlib import pytest import pytest_asyncio -from mistral_common.audio import Audio -from mistral_common.protocol.instruct.chunk import RawAudio from mistral_common.protocol.transcription.request import ( StreamingMode, TranscriptionRequest, ) +from mistral_common.tokens.tokenizers.audio import Audio from mistral_common.tokens.tokenizers.mistral import MistralTokenizer from mistral_common.tokens.tokenizers.tekken import SpecialTokenPolicy @@ -101,7 +100,7 @@ def test_voxtral_realtime_forward(audio_assets, tokenizer, engine): def from_file(file_path: str): audio = Audio.from_file(file_path, strict=False) req = TranscriptionRequest( - audio=RawAudio.from_audio(audio), + audio=audio.to_base64(audio.format), streaming=StreamingMode.OFFLINE, language=None, ) @@ -156,7 +155,7 @@ async def test_voxtral_realtime_generator(audio_assets, tokenizer, async_engine) req = TranscriptionRequest( streaming=StreamingMode.OFFLINE, - audio=RawAudio.from_audio(audio), + audio=audio.to_base64(audio.format), language=None, ) audio_enc = tokenizer.encode_transcription(req) diff --git a/vllm/model_executor/models/voxtral.py b/vllm/model_executor/models/voxtral.py index baf6d5c739..f15ec491af 100644 --- a/vllm/model_executor/models/voxtral.py +++ b/vllm/model_executor/models/voxtral.py @@ -10,11 +10,12 @@ import numpy as np import regex as re import torch import torch.nn as nn -from mistral_common.audio import Audio, mel_filter_bank -from mistral_common.protocol.instruct.chunk import AudioChunk, RawAudio, TextChunk +from mistral_common.audio import mel_filter_bank +from mistral_common.protocol.instruct.chunk import AudioChunk, TextChunk from mistral_common.protocol.instruct.messages import UserMessage from mistral_common.protocol.instruct.request import ChatCompletionRequest from mistral_common.protocol.transcription.request import TranscriptionRequest +from mistral_common.tokens.tokenizers.audio import Audio from transformers import BatchFeature, WhisperConfig from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig @@ -182,7 +183,7 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]): sampling_rate=feature_extractor.sampling_rate, format=format, ) - chunk = AudioChunk(input_audio=RawAudio.from_audio(audio_item)) + chunk = AudioChunk.from_audio(audio_item) audio_chunks.append(chunk) request = ChatCompletionRequest( @@ -462,7 +463,7 @@ class VoxtralForConditionalGeneration( audio = Audio(audio, int(stt_config.sample_rate), format="wav") # lossless req = TranscriptionRequest( model=model_config.model, - audio=RawAudio.from_audio(audio), + audio=audio.to_base64(audio.format), language=language, ) diff --git a/vllm/model_executor/models/voxtral_realtime.py b/vllm/model_executor/models/voxtral_realtime.py index 2628e1443e..8c59532e3b 100644 --- a/vllm/model_executor/models/voxtral_realtime.py +++ b/vllm/model_executor/models/voxtral_realtime.py @@ -7,13 +7,11 @@ from collections.abc import AsyncGenerator, Iterable, Iterator, Mapping import numpy as np import torch -from mistral_common.audio import Audio -from mistral_common.protocol.instruct.chunk import RawAudio from mistral_common.protocol.transcription.request import ( StreamingMode, TranscriptionRequest, ) -from mistral_common.tokens.tokenizers.audio import AudioConfig +from mistral_common.tokens.tokenizers.audio import Audio, AudioConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig @@ -477,7 +475,7 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim req = TranscriptionRequest( model=model_config.model, - audio=RawAudio.from_audio(audio), + audio=audio.to_base64(audio.format), language=language, streaming=StreamingMode.OFFLINE, )