[EPD] Support metadata-only audio inputs (#57887)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Tianyu Guo
2026-09-23 09:03:50 +08:00
committed by GitHub
co-authored by Cyrus Leung
parent 826e300cde
commit 3ee2906f47
14 changed files with 387 additions and 27 deletions
+5
View File
@@ -144,6 +144,11 @@ def test_qwen3_omni_get_updates_use_audio_in_video(
# Create a mock context
mock_ctx = Mock(spec=InputProcessingContext)
mock_ctx.tokenizer = mock_tokenizer
# `model_config` is an instance attribute, so it is not covered by the
# spec; the data parser reads it for `allow_missing_mm_embeddings`.
mock_ctx.model_config = SimpleNamespace(
multimodal_config=SimpleNamespace(allow_missing_mm_embeddings=False)
)
# Create processing info
info = Qwen3OmniMoeThinkerProcessingInfo(mock_ctx)
@@ -1,13 +1,16 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from collections.abc import Set as AbstractSet
from functools import partial
import numpy as np
import pytest
import torch
from PIL import Image
from tests.models.utils import build_model_context
from vllm.config import ModelConfig
from vllm.config.multimodal import (
AudioDummyOptions,
@@ -16,10 +19,15 @@ from vllm.config.multimodal import (
MultiModalDummyOptions,
VideoDummyOptions,
)
from vllm.distributed.ec_transfer.ec_connector.utils import (
PlaceholderMetadataResolver,
collect_ec_item_metadata,
)
from vllm.entrypoints.chat_utils import _get_embeds_data
from vllm.inputs import MultiModalDataDict, MultiModalInput
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.inputs import batched_tensors_equal
from vllm.multimodal.inputs import MultiModalFeatureSpec, batched_tensors_equal
from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
from vllm.platforms import current_platform
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
@@ -193,6 +201,79 @@ def get_token_prompt(
return inputs.prompt
@pytest.mark.parametrize(
"model_id,durations",
[
("Qwen/Qwen2-Audio-7B-Instruct", [1.0, 2.3]),
("nvidia/audio-flamingo-3-hf", [31.0]),
("fixie-ai/ultravox-v0_5-llama-3_2-1b", [1.0, 31.0]),
("Qwen/Qwen2.5-Omni-3B", [1.0, 2.3]),
("Qwen/Qwen3-Omni-30B-A3B-Instruct", [1.0, 2.3]),
],
)
def test_audio_metadata_only_roundtrip(model_id, durations, monkeypatch):
"""EC metadata preserves the raw-audio prompt without running the HF processor."""
ctx = build_model_context(model_id, limit_mm_per_prompt={"audio": len(durations)})
mm_config = ctx.model_config.multimodal_config
mm_config.enable_mm_embeds = True
mm_config.allow_missing_mm_embeddings = True
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
sample_rate = processor.info.get_feature_extractor().sampling_rate
audios = [
np.zeros(int(duration * sample_rate), dtype=np.float32)
for duration in durations
]
prompt = processor.dummy_inputs.get_dummy_text({"audio": len(audios)})
raw = processor(
prompt,
mm_items=processor.info.parse_mm_data({"audio": audios}),
hf_processor_mm_kwargs={},
)
features = [
MultiModalFeatureSpec(
data=item,
modality="audio",
identifier=str(index),
mm_position=position,
)
for index, (item, position) in enumerate(
zip(raw["mm_kwargs"]["audio"], raw["mm_placeholders"]["audio"])
)
]
published = collect_ec_item_metadata(
features, PlaceholderMetadataResolver(ctx.model_config)
)
metadata = json.loads(
json.dumps([entry["metadata"] for entry in published.values()])
)
assert len(metadata) == len(audios)
assert all(metadata)
audio_data = _get_embeds_data("audio", metadata, processor)
def unexpected_preprocessing(*args, **kwargs):
pytest.fail("The EC consumer must not preprocess audio")
monkeypatch.setattr(processor, "_call_hf_processor", unexpected_preprocessing)
remote = processor(
prompt,
mm_items=processor.info.parse_mm_data({"audio": audio_data}),
hf_processor_mm_kwargs={},
)
assert remote["prompt_token_ids"] == raw["prompt_token_ids"]
assert remote["mm_placeholders"] == raw["mm_placeholders"]
# Omni also consumes these lengths when constructing M-RoPE positions.
if "audio_feature_lengths" in audio_data:
assert torch.equal(
remote["mm_kwargs"].get_data()["audio_feature_lengths"],
raw["mm_kwargs"].get_data()["audio_feature_lengths"],
)
mm_config.allow_missing_mm_embeddings = False
ordinary_processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
with pytest.raises(ValueError, match="should contain the fields"):
ordinary_processor.info.parse_mm_data({"audio": audio_data})
def random_vision_chunk(
rng: np.random.RandomState,
min_wh: int,
+39
View File
@@ -15,6 +15,45 @@ from vllm.multimodal.parse import (
H, W = 480, 640
class AudioMetadataParser(MultiModalDataParser):
embedding_fields = {
"audio": {"audio_embeds": "values", "audio_num_tokens": "metadata"},
}
@pytest.mark.parametrize("allow_missing", [False, True])
def test_audio_metadata_requires_ec_consumer(allow_missing):
parser = AudioMetadataParser(allow_missing_mm_embeddings=allow_missing)
data = {"audio_num_tokens": torch.tensor([[3], [5]])}
if not allow_missing:
with pytest.raises(ValueError, match="audio_embeds"):
parser.parse_mm_data({"audio": data})
else:
items = parser.parse_mm_data({"audio": data})["audio"]
assert len(items) == 2
assert items.get(1)["audio_num_tokens"].item() == 5
assert items.get_processor_data() == {}
@pytest.mark.parametrize("counts", [[0], [-1], [1.5], [True], [[1, 2]]])
def test_audio_metadata_rejects_invalid_token_counts(counts):
parser = AudioMetadataParser(allow_missing_mm_embeddings=True)
with pytest.raises(ValueError, match="positive integer"):
parser.parse_mm_data({"audio": {"audio_num_tokens": torch.tensor(counts)}})
def test_audio_metadata_checks_supplied_embedding_lengths():
parser = AudioMetadataParser()
data = {
"audio_num_tokens": torch.tensor([3, 5]),
"audio_embeds": [torch.zeros(3, 8), torch.zeros(5, 8)],
}
assert len(parser.parse_mm_data({"audio": data})["audio"]) == 2
data["audio_num_tokens"] = torch.tensor([3, 4])
with pytest.raises(ValueError, match="does not match"):
parser.parse_mm_data({"audio": data})
@pytest.mark.parametrize(
"image",
[
@@ -56,6 +56,22 @@ def test_metadata_preserves_repeated_item_positions():
}
def test_audio_metadata_survives_processor_cache_hit():
resolver = Mock()
resolver.fields_for.return_value = {"audio_num_tokens"}
feature = MultiModalFeatureSpec(
data=None,
modality="audio",
identifier="audio",
mm_position=PlaceholderRange(
offset=0, length=4, is_embed=torch.tensor([False, True, True, False])
),
)
assert collect_ec_item_metadata([feature], resolver) == {
"audio": {"metadata": {"audio_num_tokens": [2]}, "item_indices": [0]}
}
@pytest.fixture
def temp_storage(tmp_path):
"""Fixture providing temporary storage path."""
@@ -87,6 +87,82 @@ class _EncoderSession:
return _EncoderResponse(self._replies.pop(0))
@pytest.mark.parametrize("second_type", ["image_url", "audio_url", "input_audio"])
@pytest.mark.parametrize("transfer", ["example", "nixl", "mooncake"])
def test_partial_audio_metadata_falls_back_with_all_transfers(
proxy, monkeypatch, second_type, transfer
):
"""Never mix metadata-only audio with raw media, or lose EC identities."""
import copy
seen = []
reported = {}
class Session:
async def post(self, url, data=None, headers=None):
body = msgspec.json.decode(data)
seen.append(body)
item = body["messages"][0]["content"][0]
mm_hash = item["uuid"] + "-processed"
entry: dict[str, Any] = {
"metadata": {"audio_feature_lengths": [100]} if len(seen) == 1 else {},
"item_indices": [0],
}
if transfer == "nixl":
entry.update(peer_host="encoder", peer_port=1234, size_bytes=4096)
reported[mm_hash] = entry
return _EncoderResponse({mm_hash: entry})
second = (
{"type": second_type, "input_audio": {"data": "large", "format": "wav"}}
if second_type == "input_audio"
else {"type": second_type, second_type: {"url": "large"}}
)
body = {
"messages": [
{
"role": "user",
"content": [
{
"type": "audio_url",
"audio_url": {"url": "small"},
"uuid": "user-audio",
},
second,
],
}
],
"mm_processor_kwargs": {"sampling_rate": 16000},
}
original = copy.deepcopy(body)
monkeypatch.setattr(proxy, "encode_session", Session())
monkeypatch.setattr(proxy, "NO_REWRITE", False)
consumer = "tcp://consumer:1" if transfer == "mooncake" else None
prepared, _, _ = asyncio.run(
proxy.prepare_for_decode(body, "r", ["http://encoder"], "", consumer)
)
assert body == original
assert prepared["mm_processor_kwargs"] == body["mm_processor_kwargs"]
content = prepared["messages"][0]["content"]
assert [item["type"] for item in content] == ["audio_url", second_type]
assert content[0]["uuid"] == "user-audio"
for item, encoded in zip(content, seen):
assert item == encoded["messages"][0]["content"][0]
params = prepared["ec_transfer_params"]
for mm_hash, entry in reported.items():
assert params[mm_hash] == entry
if transfer == "mooncake":
assert params["ec_items"] == [
{
"mm_hash": mm_hash,
"transfer_id": encoded["ec_transfer_params"]["ec_items"][0][
"transfer_id"
],
}
for mm_hash, encoded in zip(reported, seen)
]
@pytest.mark.parametrize("no_rewrite", [False, True])
@pytest.mark.parametrize(
"batch_size, expected_sizes", [(0, [3, 3]), (1, [1] * 6), (2, [2, 2, 1, 1])]
@@ -509,7 +585,11 @@ def test_video_audio_fallback(proxy, monkeypatch, no_rewrite, transfer):
original = copy.deepcopy(body)
# Collector indices describe two processed features from one video item.
video_metadata = collect_ec_item_metadata(
[SimpleNamespace(identifier=key, data=None) for key in ("video", "audio")], None
[
SimpleNamespace(identifier=key, modality=key, data=None)
for key in ("video", "audio")
],
SimpleNamespace(fields_for=lambda _: set()),
)
if transfer == "handle":
video_metadata["audio"]["transfer_id"] = "reservation"