[EPD] Support metadata-only audio inputs (#57887)
Signed-off-by: Tianyu Guo <guoty@inferact.ai> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Cyrus Leung
parent
826e300cde
commit
3ee2906f47
@@ -144,6 +144,11 @@ def test_qwen3_omni_get_updates_use_audio_in_video(
|
||||
# Create a mock context
|
||||
mock_ctx = Mock(spec=InputProcessingContext)
|
||||
mock_ctx.tokenizer = mock_tokenizer
|
||||
# `model_config` is an instance attribute, so it is not covered by the
|
||||
# spec; the data parser reads it for `allow_missing_mm_embeddings`.
|
||||
mock_ctx.model_config = SimpleNamespace(
|
||||
multimodal_config=SimpleNamespace(allow_missing_mm_embeddings=False)
|
||||
)
|
||||
|
||||
# Create processing info
|
||||
info = Qwen3OmniMoeThinkerProcessingInfo(mock_ctx)
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from collections.abc import Set as AbstractSet
|
||||
from functools import partial
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from tests.models.utils import build_model_context
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config.multimodal import (
|
||||
AudioDummyOptions,
|
||||
@@ -16,10 +19,15 @@ from vllm.config.multimodal import (
|
||||
MultiModalDummyOptions,
|
||||
VideoDummyOptions,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.utils import (
|
||||
PlaceholderMetadataResolver,
|
||||
collect_ec_item_metadata,
|
||||
)
|
||||
from vllm.entrypoints.chat_utils import _get_embeds_data
|
||||
from vllm.inputs import MultiModalDataDict, MultiModalInput
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
from vllm.multimodal.inputs import batched_tensors_equal
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, batched_tensors_equal
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
|
||||
@@ -193,6 +201,79 @@ def get_token_prompt(
|
||||
return inputs.prompt
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_id,durations",
|
||||
[
|
||||
("Qwen/Qwen2-Audio-7B-Instruct", [1.0, 2.3]),
|
||||
("nvidia/audio-flamingo-3-hf", [31.0]),
|
||||
("fixie-ai/ultravox-v0_5-llama-3_2-1b", [1.0, 31.0]),
|
||||
("Qwen/Qwen2.5-Omni-3B", [1.0, 2.3]),
|
||||
("Qwen/Qwen3-Omni-30B-A3B-Instruct", [1.0, 2.3]),
|
||||
],
|
||||
)
|
||||
def test_audio_metadata_only_roundtrip(model_id, durations, monkeypatch):
|
||||
"""EC metadata preserves the raw-audio prompt without running the HF processor."""
|
||||
ctx = build_model_context(model_id, limit_mm_per_prompt={"audio": len(durations)})
|
||||
mm_config = ctx.model_config.multimodal_config
|
||||
mm_config.enable_mm_embeds = True
|
||||
mm_config.allow_missing_mm_embeddings = True
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
sample_rate = processor.info.get_feature_extractor().sampling_rate
|
||||
audios = [
|
||||
np.zeros(int(duration * sample_rate), dtype=np.float32)
|
||||
for duration in durations
|
||||
]
|
||||
prompt = processor.dummy_inputs.get_dummy_text({"audio": len(audios)})
|
||||
raw = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data({"audio": audios}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
features = [
|
||||
MultiModalFeatureSpec(
|
||||
data=item,
|
||||
modality="audio",
|
||||
identifier=str(index),
|
||||
mm_position=position,
|
||||
)
|
||||
for index, (item, position) in enumerate(
|
||||
zip(raw["mm_kwargs"]["audio"], raw["mm_placeholders"]["audio"])
|
||||
)
|
||||
]
|
||||
published = collect_ec_item_metadata(
|
||||
features, PlaceholderMetadataResolver(ctx.model_config)
|
||||
)
|
||||
metadata = json.loads(
|
||||
json.dumps([entry["metadata"] for entry in published.values()])
|
||||
)
|
||||
assert len(metadata) == len(audios)
|
||||
assert all(metadata)
|
||||
audio_data = _get_embeds_data("audio", metadata, processor)
|
||||
|
||||
def unexpected_preprocessing(*args, **kwargs):
|
||||
pytest.fail("The EC consumer must not preprocess audio")
|
||||
|
||||
monkeypatch.setattr(processor, "_call_hf_processor", unexpected_preprocessing)
|
||||
remote = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data({"audio": audio_data}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
assert remote["prompt_token_ids"] == raw["prompt_token_ids"]
|
||||
assert remote["mm_placeholders"] == raw["mm_placeholders"]
|
||||
# Omni also consumes these lengths when constructing M-RoPE positions.
|
||||
if "audio_feature_lengths" in audio_data:
|
||||
assert torch.equal(
|
||||
remote["mm_kwargs"].get_data()["audio_feature_lengths"],
|
||||
raw["mm_kwargs"].get_data()["audio_feature_lengths"],
|
||||
)
|
||||
|
||||
mm_config.allow_missing_mm_embeddings = False
|
||||
ordinary_processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
with pytest.raises(ValueError, match="should contain the fields"):
|
||||
ordinary_processor.info.parse_mm_data({"audio": audio_data})
|
||||
|
||||
|
||||
def random_vision_chunk(
|
||||
rng: np.random.RandomState,
|
||||
min_wh: int,
|
||||
|
||||
@@ -15,6 +15,45 @@ from vllm.multimodal.parse import (
|
||||
H, W = 480, 640
|
||||
|
||||
|
||||
class AudioMetadataParser(MultiModalDataParser):
|
||||
embedding_fields = {
|
||||
"audio": {"audio_embeds": "values", "audio_num_tokens": "metadata"},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("allow_missing", [False, True])
|
||||
def test_audio_metadata_requires_ec_consumer(allow_missing):
|
||||
parser = AudioMetadataParser(allow_missing_mm_embeddings=allow_missing)
|
||||
data = {"audio_num_tokens": torch.tensor([[3], [5]])}
|
||||
if not allow_missing:
|
||||
with pytest.raises(ValueError, match="audio_embeds"):
|
||||
parser.parse_mm_data({"audio": data})
|
||||
else:
|
||||
items = parser.parse_mm_data({"audio": data})["audio"]
|
||||
assert len(items) == 2
|
||||
assert items.get(1)["audio_num_tokens"].item() == 5
|
||||
assert items.get_processor_data() == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("counts", [[0], [-1], [1.5], [True], [[1, 2]]])
|
||||
def test_audio_metadata_rejects_invalid_token_counts(counts):
|
||||
parser = AudioMetadataParser(allow_missing_mm_embeddings=True)
|
||||
with pytest.raises(ValueError, match="positive integer"):
|
||||
parser.parse_mm_data({"audio": {"audio_num_tokens": torch.tensor(counts)}})
|
||||
|
||||
|
||||
def test_audio_metadata_checks_supplied_embedding_lengths():
|
||||
parser = AudioMetadataParser()
|
||||
data = {
|
||||
"audio_num_tokens": torch.tensor([3, 5]),
|
||||
"audio_embeds": [torch.zeros(3, 8), torch.zeros(5, 8)],
|
||||
}
|
||||
assert len(parser.parse_mm_data({"audio": data})["audio"]) == 2
|
||||
data["audio_num_tokens"] = torch.tensor([3, 4])
|
||||
with pytest.raises(ValueError, match="does not match"):
|
||||
parser.parse_mm_data({"audio": data})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"image",
|
||||
[
|
||||
|
||||
@@ -56,6 +56,22 @@ def test_metadata_preserves_repeated_item_positions():
|
||||
}
|
||||
|
||||
|
||||
def test_audio_metadata_survives_processor_cache_hit():
|
||||
resolver = Mock()
|
||||
resolver.fields_for.return_value = {"audio_num_tokens"}
|
||||
feature = MultiModalFeatureSpec(
|
||||
data=None,
|
||||
modality="audio",
|
||||
identifier="audio",
|
||||
mm_position=PlaceholderRange(
|
||||
offset=0, length=4, is_embed=torch.tensor([False, True, True, False])
|
||||
),
|
||||
)
|
||||
assert collect_ec_item_metadata([feature], resolver) == {
|
||||
"audio": {"metadata": {"audio_num_tokens": [2]}, "item_indices": [0]}
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def temp_storage(tmp_path):
|
||||
"""Fixture providing temporary storage path."""
|
||||
|
||||
@@ -87,6 +87,82 @@ class _EncoderSession:
|
||||
return _EncoderResponse(self._replies.pop(0))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("second_type", ["image_url", "audio_url", "input_audio"])
|
||||
@pytest.mark.parametrize("transfer", ["example", "nixl", "mooncake"])
|
||||
def test_partial_audio_metadata_falls_back_with_all_transfers(
|
||||
proxy, monkeypatch, second_type, transfer
|
||||
):
|
||||
"""Never mix metadata-only audio with raw media, or lose EC identities."""
|
||||
import copy
|
||||
|
||||
seen = []
|
||||
reported = {}
|
||||
|
||||
class Session:
|
||||
async def post(self, url, data=None, headers=None):
|
||||
body = msgspec.json.decode(data)
|
||||
seen.append(body)
|
||||
item = body["messages"][0]["content"][0]
|
||||
mm_hash = item["uuid"] + "-processed"
|
||||
entry: dict[str, Any] = {
|
||||
"metadata": {"audio_feature_lengths": [100]} if len(seen) == 1 else {},
|
||||
"item_indices": [0],
|
||||
}
|
||||
if transfer == "nixl":
|
||||
entry.update(peer_host="encoder", peer_port=1234, size_bytes=4096)
|
||||
reported[mm_hash] = entry
|
||||
return _EncoderResponse({mm_hash: entry})
|
||||
|
||||
second = (
|
||||
{"type": second_type, "input_audio": {"data": "large", "format": "wav"}}
|
||||
if second_type == "input_audio"
|
||||
else {"type": second_type, second_type: {"url": "large"}}
|
||||
)
|
||||
body = {
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": {"url": "small"},
|
||||
"uuid": "user-audio",
|
||||
},
|
||||
second,
|
||||
],
|
||||
}
|
||||
],
|
||||
"mm_processor_kwargs": {"sampling_rate": 16000},
|
||||
}
|
||||
original = copy.deepcopy(body)
|
||||
monkeypatch.setattr(proxy, "encode_session", Session())
|
||||
monkeypatch.setattr(proxy, "NO_REWRITE", False)
|
||||
consumer = "tcp://consumer:1" if transfer == "mooncake" else None
|
||||
prepared, _, _ = asyncio.run(
|
||||
proxy.prepare_for_decode(body, "r", ["http://encoder"], "", consumer)
|
||||
)
|
||||
assert body == original
|
||||
assert prepared["mm_processor_kwargs"] == body["mm_processor_kwargs"]
|
||||
content = prepared["messages"][0]["content"]
|
||||
assert [item["type"] for item in content] == ["audio_url", second_type]
|
||||
assert content[0]["uuid"] == "user-audio"
|
||||
for item, encoded in zip(content, seen):
|
||||
assert item == encoded["messages"][0]["content"][0]
|
||||
params = prepared["ec_transfer_params"]
|
||||
for mm_hash, entry in reported.items():
|
||||
assert params[mm_hash] == entry
|
||||
if transfer == "mooncake":
|
||||
assert params["ec_items"] == [
|
||||
{
|
||||
"mm_hash": mm_hash,
|
||||
"transfer_id": encoded["ec_transfer_params"]["ec_items"][0][
|
||||
"transfer_id"
|
||||
],
|
||||
}
|
||||
for mm_hash, encoded in zip(reported, seen)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("no_rewrite", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"batch_size, expected_sizes", [(0, [3, 3]), (1, [1] * 6), (2, [2, 2, 1, 1])]
|
||||
@@ -509,7 +585,11 @@ def test_video_audio_fallback(proxy, monkeypatch, no_rewrite, transfer):
|
||||
original = copy.deepcopy(body)
|
||||
# Collector indices describe two processed features from one video item.
|
||||
video_metadata = collect_ec_item_metadata(
|
||||
[SimpleNamespace(identifier=key, data=None) for key in ("video", "audio")], None
|
||||
[
|
||||
SimpleNamespace(identifier=key, modality=key, data=None)
|
||||
for key in ("video", "audio")
|
||||
],
|
||||
SimpleNamespace(fields_for=lambda _: set()),
|
||||
)
|
||||
if transfer == "handle":
|
||||
video_metadata["audio"]["transfer_id"] = "reservation"
|
||||
|
||||
Reference in New Issue
Block a user