[1/N] Harden Transformers modelling backend multi-modal path (#51408)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-08-07 17:30:43 +00:00
committed by GitHub
parent fcde8e1460
commit 45273b8dcb
4 changed files with 204 additions and 52 deletions
@@ -134,6 +134,17 @@ def get_model_ids_to_test():
return _get_model_ids_to_test(vllm_only_archs)
def get_transformers_backend_model_ids_to_test():
return sorted(
{
model_id
for arch, info in _TRANSFORMERS_BACKEND_MODELS.items()
if "MultiModal" in arch
for model_id in (info.default, *info.extras.values())
}
)
def get_text_token_prompts(
processor: BaseMultiModalProcessor,
mm_data: MultiModalDataDict,
@@ -211,6 +222,7 @@ def _test_processing_correctness(
hit_rate: float,
num_batches: int,
simplify_rate: float,
model_impl: str = "auto",
):
if model_id_or_arch in HF_EXAMPLE_MODELS.get_supported_archs():
# Use model architecture to get the default model id
@@ -238,6 +250,7 @@ def _test_processing_correctness(
enable_mm_embeds=model_info.require_embed_inputs,
enforce_eager=model_info.enforce_eager,
dtype=model_info.dtype,
model_impl=model_impl,
)
# Ensure that the cache can fit all of the data
# (set after because ModelConfig would set it to 0 for encoder-decoder models)
@@ -480,6 +493,25 @@ def test_processing_correctness(
)
@pytest.mark.parametrize("model_id", get_transformers_backend_model_ids_to_test())
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
@pytest.mark.parametrize("num_batches", [32])
@pytest.mark.parametrize("simplify_rate", [1.0])
def test_processing_correctness_transformers(
model_id: str,
hit_rate: float,
num_batches: int,
simplify_rate: float,
):
_test_processing_correctness(
model_id,
hit_rate=hit_rate,
num_batches=num_batches,
simplify_rate=simplify_rate,
model_impl="transformers",
)
def _assert_inputs_equal(
a: MultiModalInput,
b: MultiModalInput,