From a36dfc93cb1e4ca11425cfde48b5ed2264027aa2 Mon Sep 17 00:00:00 2001 From: Jiatai Wang Date: Thu, 10 Sep 2026 12:20:03 -0700 Subject: [PATCH] [Bugfix][Multimodal] Restore cached audio inputs with UUIDs (#56310) Signed-off-by: JiataiWang Co-authored-by: OpenAI Codex --- .../entrypoints/unit_tests/test_chat_utils.py | 8 ++++++-- tests/multimodal/test_parse.py | 18 ++++++++++++++++++ vllm/entrypoints/chat_utils.py | 2 +- vllm/multimodal/parse.py | 9 +++++++-- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/tests/entrypoints/unit_tests/test_chat_utils.py b/tests/entrypoints/unit_tests/test_chat_utils.py index 575bb904d6..630f641df5 100644 --- a/tests/entrypoints/unit_tests/test_chat_utils.py +++ b/tests/entrypoints/unit_tests/test_chat_utils.py @@ -2384,8 +2384,10 @@ def test_parse_chat_messages_include_thinking_chunk(mistral_model_config): assert conversation_with_thinking == expected_conversation +@pytest.mark.parametrize("input_audio", [None, {}, {"data": "", "format": "wav"}]) def test_parse_chat_messages_single_empty_audio_with_uuid( qwen2_audio_model_config, + input_audio, ): audio_uuid = "abcd" conversation, mm_data, mm_uuids = parse_chat_messages( @@ -2395,7 +2397,7 @@ def test_parse_chat_messages_single_empty_audio_with_uuid( "content": [ { "type": "input_audio", - "input_audio": {}, + "input_audio": input_audio, "uuid": audio_uuid, }, {"type": "text", "text": "What does the audio say?"}, @@ -2418,8 +2420,10 @@ def test_parse_chat_messages_single_empty_audio_with_uuid( @pytest.mark.asyncio +@pytest.mark.parametrize("input_audio", [None, {}, {"data": "", "format": "wav"}]) async def test_parse_chat_messages_single_empty_audio_with_uuid_async( qwen2_audio_model_config, + input_audio, ): audio_uuid = "abcd" conversation, mm_data, mm_uuids = await parse_chat_messages_async( @@ -2429,7 +2433,7 @@ async def test_parse_chat_messages_single_empty_audio_with_uuid_async( "content": [ { "type": "input_audio", - "input_audio": {}, + "input_audio": input_audio, "uuid": audio_uuid, }, {"type": "text", "text": "What does the audio say?"}, diff --git a/tests/multimodal/test_parse.py b/tests/multimodal/test_parse.py index 685e615f78..c34c8eec8b 100644 --- a/tests/multimodal/test_parse.py +++ b/tests/multimodal/test_parse.py @@ -6,6 +6,7 @@ import torch from PIL import Image from vllm.multimodal.parse import ( + AudioProcessorItems, ImageProcessorItems, MultiModalDataParser, VideoProcessorItems, @@ -58,6 +59,7 @@ def test_frame_size_hwc_chw(frame): @pytest.mark.parametrize( "modality,processor_cls", [ + ("audio", AudioProcessorItems), ("image", ImageProcessorItems), ("video", VideoProcessorItems), ], @@ -68,3 +70,19 @@ def test_parse_mm_data_accepts_none_cached_item(modality, processor_cls): assert isinstance(items, processor_cls) assert len(items) == 1 assert items.get(0) is None + + +def test_cached_audio_items_preserve_positions_during_resampling(): + waveform = np.arange(16, dtype=np.float32) + parser = MultiModalDataParser( + target_sr=16000, target_channels=1, audio_resample_method="scipy" + ) + items = parser.parse_mm_data( + {"audio": [None, (waveform, 8000), None, (waveform, 16000)]} + )["audio"] + + assert len(items) == 4 + assert items.get(0) is None + assert items.get(2) is None + assert len(items.get(1)) == 32 + np.testing.assert_array_equal(items.get(3), waveform) diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index 327f9c4b24..addcc5f76d 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -1773,7 +1773,7 @@ def _parse_chat_message_content_mm_part( # with url as a dict of {"url": url} audio_url = audio_url.get("url", None) return "audio_url", audio_url - if part.get("input_audio") is not None: + if "input_audio" in part: input_audio_params = _InputAudioParser(part).get("input_audio", None) return "input_audio", input_audio_params if "video_url" in part: diff --git a/vllm/multimodal/parse.py b/vllm/multimodal/parse.py index d96f19f39b..fefaeef2b0 100644 --- a/vllm/multimodal/parse.py +++ b/vllm/multimodal/parse.py @@ -678,7 +678,7 @@ class MultiModalDataParser: if self.is_embeddings(data): return AudioEmbeddingItems(data, self.expected_hidden_size) - data_items: list[AudioItem] + data_items: list[AudioItem | None] if ( (is_list_of(data, float) and len(data) > 0) or (isinstance(data, (np.ndarray, torch.Tensor)) and data.ndim == 1) @@ -690,8 +690,13 @@ class MultiModalDataParser: else: data_items = data # type: ignore[assignment] - new_audios = list[np.ndarray]() + new_audios = list[np.ndarray | None]() for data_item in data_items: + # Requests can omit audio samples when reusing a cached UUID. + if data_item is None: + new_audios.append(None) + continue + audio, orig_sr = self._get_audio_with_sr(data_item) if orig_sr is None: new_audio = audio