[Bugfix][Multimodal] Restore cached audio inputs with UUIDs (#56310)
Signed-off-by: JiataiWang <wangjiatai@proton.me> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
7de70fa7ae
commit
a36dfc93cb
@@ -2384,8 +2384,10 @@ def test_parse_chat_messages_include_thinking_chunk(mistral_model_config):
|
||||
assert conversation_with_thinking == expected_conversation
|
||||
|
||||
|
||||
@pytest.mark.parametrize("input_audio", [None, {}, {"data": "", "format": "wav"}])
|
||||
def test_parse_chat_messages_single_empty_audio_with_uuid(
|
||||
qwen2_audio_model_config,
|
||||
input_audio,
|
||||
):
|
||||
audio_uuid = "abcd"
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
@@ -2395,7 +2397,7 @@ def test_parse_chat_messages_single_empty_audio_with_uuid(
|
||||
"content": [
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": {},
|
||||
"input_audio": input_audio,
|
||||
"uuid": audio_uuid,
|
||||
},
|
||||
{"type": "text", "text": "What does the audio say?"},
|
||||
@@ -2418,8 +2420,10 @@ def test_parse_chat_messages_single_empty_audio_with_uuid(
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("input_audio", [None, {}, {"data": "", "format": "wav"}])
|
||||
async def test_parse_chat_messages_single_empty_audio_with_uuid_async(
|
||||
qwen2_audio_model_config,
|
||||
input_audio,
|
||||
):
|
||||
audio_uuid = "abcd"
|
||||
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
|
||||
@@ -2429,7 +2433,7 @@ async def test_parse_chat_messages_single_empty_audio_with_uuid_async(
|
||||
"content": [
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": {},
|
||||
"input_audio": input_audio,
|
||||
"uuid": audio_uuid,
|
||||
},
|
||||
{"type": "text", "text": "What does the audio say?"},
|
||||
|
||||
@@ -6,6 +6,7 @@ import torch
|
||||
from PIL import Image
|
||||
|
||||
from vllm.multimodal.parse import (
|
||||
AudioProcessorItems,
|
||||
ImageProcessorItems,
|
||||
MultiModalDataParser,
|
||||
VideoProcessorItems,
|
||||
@@ -58,6 +59,7 @@ def test_frame_size_hwc_chw(frame):
|
||||
@pytest.mark.parametrize(
|
||||
"modality,processor_cls",
|
||||
[
|
||||
("audio", AudioProcessorItems),
|
||||
("image", ImageProcessorItems),
|
||||
("video", VideoProcessorItems),
|
||||
],
|
||||
@@ -68,3 +70,19 @@ def test_parse_mm_data_accepts_none_cached_item(modality, processor_cls):
|
||||
assert isinstance(items, processor_cls)
|
||||
assert len(items) == 1
|
||||
assert items.get(0) is None
|
||||
|
||||
|
||||
def test_cached_audio_items_preserve_positions_during_resampling():
|
||||
waveform = np.arange(16, dtype=np.float32)
|
||||
parser = MultiModalDataParser(
|
||||
target_sr=16000, target_channels=1, audio_resample_method="scipy"
|
||||
)
|
||||
items = parser.parse_mm_data(
|
||||
{"audio": [None, (waveform, 8000), None, (waveform, 16000)]}
|
||||
)["audio"]
|
||||
|
||||
assert len(items) == 4
|
||||
assert items.get(0) is None
|
||||
assert items.get(2) is None
|
||||
assert len(items.get(1)) == 32
|
||||
np.testing.assert_array_equal(items.get(3), waveform)
|
||||
|
||||
@@ -1773,7 +1773,7 @@ def _parse_chat_message_content_mm_part(
|
||||
# with url as a dict of {"url": url}
|
||||
audio_url = audio_url.get("url", None)
|
||||
return "audio_url", audio_url
|
||||
if part.get("input_audio") is not None:
|
||||
if "input_audio" in part:
|
||||
input_audio_params = _InputAudioParser(part).get("input_audio", None)
|
||||
return "input_audio", input_audio_params
|
||||
if "video_url" in part:
|
||||
|
||||
@@ -678,7 +678,7 @@ class MultiModalDataParser:
|
||||
if self.is_embeddings(data):
|
||||
return AudioEmbeddingItems(data, self.expected_hidden_size)
|
||||
|
||||
data_items: list[AudioItem]
|
||||
data_items: list[AudioItem | None]
|
||||
if (
|
||||
(is_list_of(data, float) and len(data) > 0)
|
||||
or (isinstance(data, (np.ndarray, torch.Tensor)) and data.ndim == 1)
|
||||
@@ -690,8 +690,13 @@ class MultiModalDataParser:
|
||||
else:
|
||||
data_items = data # type: ignore[assignment]
|
||||
|
||||
new_audios = list[np.ndarray]()
|
||||
new_audios = list[np.ndarray | None]()
|
||||
for data_item in data_items:
|
||||
# Requests can omit audio samples when reusing a cached UUID.
|
||||
if data_item is None:
|
||||
new_audios.append(None)
|
||||
continue
|
||||
|
||||
audio, orig_sr = self._get_audio_with_sr(data_item)
|
||||
if orig_sr is None:
|
||||
new_audio = audio
|
||||
|
||||
Reference in New Issue
Block a user