[CI/Build] Avoid duplicate runner startup for multimodal test (#52417)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -94,7 +94,8 @@ def get_model_type_cases(
|
||||
test_info.needs_video_metadata
|
||||
)
|
||||
|
||||
# No sizes passed for custom inputs, since inputs are directly provided
|
||||
# Keep all size batches in one test case so they share the same model
|
||||
# instances. No sizes are passed for preprocessed audio or custom inputs.
|
||||
if test_type not in (
|
||||
VLMTestType.CUSTOM_INPUTS,
|
||||
VLMTestType.AUDIO,
|
||||
@@ -102,7 +103,9 @@ def get_model_type_cases(
|
||||
wrapped_sizes = get_wrapped_test_sizes(test_info, test_type)
|
||||
if wrapped_sizes is None:
|
||||
raise ValueError(f"Sizes must be set for test type {test_type}")
|
||||
iter_kwargs["size_wrapper"] = wrapped_sizes
|
||||
if not wrapped_sizes:
|
||||
return []
|
||||
iter_kwargs["size_wrappers"] = (wrapped_sizes,)
|
||||
|
||||
# Otherwise expand the custom test options instead
|
||||
elif test_type == VLMTestType.CUSTOM_INPUTS:
|
||||
@@ -127,9 +130,8 @@ def get_parametrized_options(
|
||||
create_new_process_for_each_test: bool,
|
||||
):
|
||||
"""Converts all of our VLMTestInfo into an expanded list of parameters.
|
||||
This is similar to nesting pytest parametrize calls, but done directly
|
||||
through an itertools product so that each test can set things like
|
||||
size factors etc, while still running in isolated test cases.
|
||||
Runner configuration values are expanded through an itertools product.
|
||||
Input size batches stay grouped so they can share model instances.
|
||||
"""
|
||||
matching_tests = get_filtered_test_settings(
|
||||
test_settings, test_type, create_new_process_for_each_test
|
||||
@@ -149,8 +151,7 @@ def get_wrapped_test_sizes(
|
||||
test_info: VLMTestInfo, test_type: VLMTestType
|
||||
) -> tuple[ImageSizeWrapper, ...]:
|
||||
"""Given a test info which may have size factors or fixed sizes, wrap them
|
||||
and combine them into an iterable, each of which will be used in parameter
|
||||
expansion.
|
||||
and combine them into an iterable of request batches.
|
||||
|
||||
Args:
|
||||
test_info: Test configuration to be expanded.
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
types / modalities.
|
||||
"""
|
||||
|
||||
import itertools
|
||||
from pathlib import PosixPath
|
||||
|
||||
from .....conftest import (
|
||||
@@ -27,9 +28,14 @@ def run_single_image_test(
|
||||
vllm_runner: type[VllmRunner],
|
||||
image_assets: ImageTestAssets,
|
||||
):
|
||||
assert test_case.size_wrapper is not None
|
||||
inputs = builders.build_single_image_inputs_from_test_info(
|
||||
model_test_info, image_assets, test_case.size_wrapper, tmp_path
|
||||
assert test_case.size_wrappers
|
||||
inputs = list(
|
||||
itertools.chain.from_iterable(
|
||||
builders.build_single_image_inputs_from_test_info(
|
||||
model_test_info, image_assets, size_wrapper, tmp_path
|
||||
)
|
||||
for size_wrapper in test_case.size_wrappers
|
||||
)
|
||||
)
|
||||
|
||||
core.run_test(
|
||||
@@ -55,9 +61,14 @@ def run_multi_image_test(
|
||||
vllm_runner: type[VllmRunner],
|
||||
image_assets: ImageTestAssets,
|
||||
):
|
||||
assert test_case.size_wrapper is not None
|
||||
inputs = builders.build_multi_image_inputs_from_test_info(
|
||||
model_test_info, image_assets, test_case.size_wrapper, tmp_path
|
||||
assert test_case.size_wrappers
|
||||
inputs = list(
|
||||
itertools.chain.from_iterable(
|
||||
builders.build_multi_image_inputs_from_test_info(
|
||||
model_test_info, image_assets, size_wrapper, tmp_path
|
||||
)
|
||||
for size_wrapper in test_case.size_wrappers
|
||||
)
|
||||
)
|
||||
|
||||
core.run_test(
|
||||
@@ -82,9 +93,20 @@ def run_embedding_test(
|
||||
vllm_runner: type[VllmRunner],
|
||||
image_assets: ImageTestAssets,
|
||||
):
|
||||
assert test_case.size_wrapper is not None
|
||||
inputs, vllm_embeddings = builders.build_embedding_inputs_from_test_info(
|
||||
model_test_info, image_assets, test_case.size_wrapper
|
||||
assert test_case.size_wrappers
|
||||
inputs_and_embeddings = [
|
||||
builders.build_embedding_inputs_from_test_info(
|
||||
model_test_info, image_assets, size_wrapper
|
||||
)
|
||||
for size_wrapper in test_case.size_wrappers
|
||||
]
|
||||
inputs = list(
|
||||
itertools.chain.from_iterable(inputs for inputs, _ in inputs_and_embeddings)
|
||||
)
|
||||
vllm_embeddings = list(
|
||||
itertools.chain.from_iterable(
|
||||
embeddings for _, embeddings in inputs_and_embeddings
|
||||
)
|
||||
)
|
||||
|
||||
core.run_test(
|
||||
@@ -110,14 +132,19 @@ def run_video_test(
|
||||
vllm_runner: type[VllmRunner],
|
||||
video_assets: VideoTestAssets,
|
||||
):
|
||||
assert test_case.size_wrapper is not None
|
||||
assert test_case.size_wrappers
|
||||
assert test_case.num_video_frames is not None
|
||||
inputs = builders.build_video_inputs_from_test_info(
|
||||
model_test_info,
|
||||
video_assets,
|
||||
test_case.size_wrapper,
|
||||
test_case.num_video_frames,
|
||||
test_case.needs_video_metadata,
|
||||
inputs = list(
|
||||
itertools.chain.from_iterable(
|
||||
builders.build_video_inputs_from_test_info(
|
||||
model_test_info,
|
||||
video_assets,
|
||||
size_wrapper,
|
||||
test_case.num_video_frames,
|
||||
test_case.needs_video_metadata,
|
||||
)
|
||||
for size_wrapper in test_case.size_wrappers
|
||||
)
|
||||
)
|
||||
|
||||
core.run_test(
|
||||
|
||||
@@ -158,11 +158,9 @@ class VLMTestInfo(NamedTuple):
|
||||
num_video_frames: int | tuple[int] = 16
|
||||
needs_video_metadata: bool = False
|
||||
|
||||
# Fixed image sizes / image size factors; most tests use image_size_factors
|
||||
# The values provided for these two fields will be stacked and expanded
|
||||
# such that each model will consider each image size factor / image size
|
||||
# once per tests (much like concatenating and wrapping in one parametrize
|
||||
# call)
|
||||
# Fixed image sizes / image size factors; most tests use image_size_factors.
|
||||
# Each inner iterable defines one request batch. All batches are run against
|
||||
# the same model instance.
|
||||
image_size_factors: Iterable[Iterable[float]] = IMAGE_SIZE_FACTORS
|
||||
image_sizes: Iterable[Iterable[tuple[int, int]]] | None = None
|
||||
|
||||
@@ -211,8 +209,8 @@ class ExpandableVLMTestArgs(NamedTuple):
|
||||
num_logprobs: int
|
||||
dtype: str
|
||||
distributed_executor_backend: str | None
|
||||
# Sizes are used for everything except for custom input tests
|
||||
size_wrapper: ImageSizeWrapper | None = None
|
||||
# Sizes are used for everything except audio and custom input tests.
|
||||
size_wrappers: tuple[ImageSizeWrapper, ...] = ()
|
||||
# Video only
|
||||
num_video_frames: int | None = None
|
||||
needs_video_metadata: bool = False
|
||||
|
||||
Reference in New Issue
Block a user