[3/N] Harden Transformers modelling backend multi-modal path (#51827)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -4,8 +4,11 @@ import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.model_executor.models.transformers.multimodal import LegacyMultiModalProcessor
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from .transformers_backend import PROCESSOR_CLASSES, create_processor
|
||||
|
||||
AUDIO_MODEL_SETTINGS = {
|
||||
"ibm-granite/granite-speech-3.3-2b": {
|
||||
"prompt": (
|
||||
@@ -53,6 +56,7 @@ AUDIO_MODEL_SETTINGS = {
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
@pytest.mark.parametrize(
|
||||
"model_id",
|
||||
[
|
||||
@@ -61,12 +65,12 @@ AUDIO_MODEL_SETTINGS = {
|
||||
pytest.param(
|
||||
"mistralai/Voxtral-Mini-3B-2507",
|
||||
marks=pytest.mark.xfail(
|
||||
reason="MistralCommonBackend.encode does not produce the audio "
|
||||
"placeholder token (ID 24) from raw text. apply_chat_template "
|
||||
"yields token IDs with placeholders, but MultiModalProcessor."
|
||||
"apply() decodes the prompt back to text and re-tokenizes, at "
|
||||
"which point the placeholders are lost. Fix belongs in "
|
||||
"mistral_common or in the Voxtral-specific path.",
|
||||
reason="Voxtral's mistral_common processor does not compose with "
|
||||
"the Transformers modelling backend. Loading it currently fails "
|
||||
"outright, because MistralCommonBackend.from_pretrained rejects "
|
||||
"the kwargs vLLM passes, and it implements no "
|
||||
"`replace_audio_token`, so it would report no replacement "
|
||||
"offsets. Both fixes belong in mistral_common or transformers.",
|
||||
strict=False,
|
||||
),
|
||||
),
|
||||
@@ -74,15 +78,10 @@ AUDIO_MODEL_SETTINGS = {
|
||||
"zai-org/GLM-ASR-Nano-2512",
|
||||
],
|
||||
)
|
||||
def test_audio_multimodal_processor(model_id):
|
||||
def test_audio_multimodal_processor(model_id, processor_cls):
|
||||
settings = AUDIO_MODEL_SETTINGS[model_id]
|
||||
|
||||
model_config = ModelConfig(
|
||||
model=model_id,
|
||||
model_impl="transformers",
|
||||
)
|
||||
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
|
||||
audio = np.zeros(16000, dtype=np.float32)
|
||||
mm_data = {"audio": (audio, 16000)}
|
||||
@@ -113,10 +112,13 @@ def test_audio_multimodal_processor(model_id):
|
||||
)
|
||||
|
||||
|
||||
def _process_granite_speech(separator: str):
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
@pytest.mark.parametrize("separator", [" and ", ""])
|
||||
def test_audio_multiple_inputs(separator, processor_cls):
|
||||
"""Multiple audios per prompt are each detected as a separate placeholder
|
||||
and multi-modal item by the Transformers modelling backend."""
|
||||
model_id = "ibm-granite/granite-speech-3.3-2b"
|
||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
|
||||
audio_token = mm_processor.info.get_hf_processor().audio_token
|
||||
# One token per audio; the processor expands each to its placeholder run.
|
||||
@@ -126,17 +128,21 @@ def _process_granite_speech(separator: str):
|
||||
)
|
||||
audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)]
|
||||
|
||||
return mm_processor(
|
||||
prompt=prompt,
|
||||
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
def process():
|
||||
return mm_processor(
|
||||
prompt=prompt,
|
||||
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
# The legacy path reads placeholders off contiguous runs of the audio token, so
|
||||
# it cannot tell adjacent ones apart and says so instead of merging them
|
||||
if processor_cls is LegacyMultiModalProcessor and not separator:
|
||||
with pytest.raises(ValueError, match="Separate them in the prompt"):
|
||||
process()
|
||||
return
|
||||
|
||||
def test_audio_multiple_inputs():
|
||||
"""Multiple audios per prompt are each detected as a separate placeholder
|
||||
and multi-modal item by the Transformers modelling backend."""
|
||||
result = _process_granite_speech(separator=" and ")
|
||||
result = process()
|
||||
|
||||
assert len(result["mm_placeholders"]["audio"]) == 2
|
||||
assert len(result["mm_kwargs"]["audio"]) == 2
|
||||
@@ -167,9 +173,3 @@ def test_unclaimed_fields_warn_rather_than_raise():
|
||||
|
||||
assert owned["audio"] == ["input_features"]
|
||||
assert owned["image"] == []
|
||||
|
||||
|
||||
def test_audio_adjacent_inputs():
|
||||
"""Adjacent audios are rejected rather than silently merged into one placeholder."""
|
||||
with pytest.raises(ValueError, match="told apart"):
|
||||
_process_granite_speech(separator="")
|
||||
|
||||
Reference in New Issue
Block a user