[1/N] Harden Transformers modelling backend multi-modal path (#51408)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -134,6 +134,17 @@ def get_model_ids_to_test():
|
||||
return _get_model_ids_to_test(vllm_only_archs)
|
||||
|
||||
|
||||
def get_transformers_backend_model_ids_to_test():
|
||||
return sorted(
|
||||
{
|
||||
model_id
|
||||
for arch, info in _TRANSFORMERS_BACKEND_MODELS.items()
|
||||
if "MultiModal" in arch
|
||||
for model_id in (info.default, *info.extras.values())
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def get_text_token_prompts(
|
||||
processor: BaseMultiModalProcessor,
|
||||
mm_data: MultiModalDataDict,
|
||||
@@ -211,6 +222,7 @@ def _test_processing_correctness(
|
||||
hit_rate: float,
|
||||
num_batches: int,
|
||||
simplify_rate: float,
|
||||
model_impl: str = "auto",
|
||||
):
|
||||
if model_id_or_arch in HF_EXAMPLE_MODELS.get_supported_archs():
|
||||
# Use model architecture to get the default model id
|
||||
@@ -238,6 +250,7 @@ def _test_processing_correctness(
|
||||
enable_mm_embeds=model_info.require_embed_inputs,
|
||||
enforce_eager=model_info.enforce_eager,
|
||||
dtype=model_info.dtype,
|
||||
model_impl=model_impl,
|
||||
)
|
||||
# Ensure that the cache can fit all of the data
|
||||
# (set after because ModelConfig would set it to 0 for encoder-decoder models)
|
||||
@@ -480,6 +493,25 @@ def test_processing_correctness(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", get_transformers_backend_model_ids_to_test())
|
||||
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
|
||||
@pytest.mark.parametrize("num_batches", [32])
|
||||
@pytest.mark.parametrize("simplify_rate", [1.0])
|
||||
def test_processing_correctness_transformers(
|
||||
model_id: str,
|
||||
hit_rate: float,
|
||||
num_batches: int,
|
||||
simplify_rate: float,
|
||||
):
|
||||
_test_processing_correctness(
|
||||
model_id,
|
||||
hit_rate=hit_rate,
|
||||
num_batches=num_batches,
|
||||
simplify_rate=simplify_rate,
|
||||
model_impl="transformers",
|
||||
)
|
||||
|
||||
|
||||
def _assert_inputs_equal(
|
||||
a: MultiModalInput,
|
||||
b: MultiModalInput,
|
||||
|
||||
Reference in New Issue
Block a user