[CI/Build] Avoid duplicate runner startup for multimodal test (#52417)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-08-16 22:17:25 +00:00
committed by GitHub
parent 6b0b850a8b
commit e3c1cb54fc
5 changed files with 77 additions and 93 deletions
@@ -66,6 +66,11 @@ def vllm_to_hf_output(
target_dtype = "half" target_dtype = "half"
IMAGE_SIZE_FACTOR_GROUPS = (
(1.0,),
(1.0, 1.0, 1.0),
(0.25, 0.5, 1.0),
)
def run_test( def run_test(
@@ -167,17 +172,6 @@ def run_test(
@pytest.mark.parametrize("model", models) @pytest.mark.parametrize("model", models)
@pytest.mark.parametrize(
"size_factors",
[
# Single-scale
[1.0],
# Single-scale, batched
[1.0, 1.0, 1.0],
# Multi-scale
[0.25, 0.5, 1.0],
],
)
@pytest.mark.parametrize("dtype", [target_dtype]) @pytest.mark.parametrize("dtype", [target_dtype])
@pytest.mark.parametrize("max_model_len", [12800]) @pytest.mark.parametrize("max_model_len", [12800])
@pytest.mark.parametrize("max_tokens", [128]) @pytest.mark.parametrize("max_tokens", [128])
@@ -187,7 +181,6 @@ def test_models(
vllm_runner, vllm_runner,
image_assets, image_assets,
model, model,
size_factors,
dtype: str, dtype: str,
max_model_len: int, max_model_len: int,
max_tokens: int, max_tokens: int,
@@ -201,6 +194,7 @@ def test_models(
[rescale_image_size(image, factor) for factor in size_factors], [rescale_image_size(image, factor) for factor in size_factors],
None, None,
) )
for size_factors in IMAGE_SIZE_FACTOR_GROUPS
for image, prompt in zip(images, HF_IMAGE_PROMPTS) for image, prompt in zip(images, HF_IMAGE_PROMPTS)
] ]
@@ -220,19 +214,6 @@ def test_models(
@large_gpu_test(min_gb=48) @large_gpu_test(min_gb=48)
@pytest.mark.parametrize("model", models) @pytest.mark.parametrize("model", models)
@pytest.mark.parametrize(
"size_factors",
[
# No image
# [],
# Single-scale
[1.0],
# Single-scale, batched
[1.0, 1.0, 1.0],
# Multi-scale
[0.25, 0.5, 1.0],
],
)
@pytest.mark.parametrize("dtype", [target_dtype]) @pytest.mark.parametrize("dtype", [target_dtype])
@pytest.mark.parametrize("max_model_len", [25600]) @pytest.mark.parametrize("max_model_len", [25600])
@pytest.mark.parametrize("max_tokens", [128]) @pytest.mark.parametrize("max_tokens", [128])
@@ -242,7 +223,6 @@ def test_multi_images_models(
vllm_runner, vllm_runner,
image_assets, image_assets,
model, model,
size_factors,
dtype: str, dtype: str,
max_model_len: int, max_model_len: int,
max_tokens: int, max_tokens: int,
@@ -258,7 +238,8 @@ def test_multi_images_models(
for factor in size_factors for factor in size_factors
], ],
None, None,
), )
for size_factors in IMAGE_SIZE_FACTOR_GROUPS
] ]
run_test( run_test(
@@ -29,6 +29,16 @@ def enable_pickle(monkeypatch):
models = ["Qwen/Qwen2-VL-2B-Instruct"] models = ["Qwen/Qwen2-VL-2B-Instruct"]
target_dtype = "half" target_dtype = "half"
IMAGE_SIZE_FACTOR_GROUPS = (
(0.5,),
(0.5, 0.5),
(0.25, 0.5, 0.5),
)
VIDEO_SIZE_FACTOR_GROUPS = (
(0.5,),
(0.5, 0.5),
(0.25, 0.25, 0.5),
)
IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>" IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>"
VIDEO_PLACEHOLDER = "<|vision_start|><|video_pad|><|vision_end|>" VIDEO_PLACEHOLDER = "<|vision_start|><|video_pad|><|vision_end|>"
@@ -323,17 +333,6 @@ def run_embedding_input_test(
@pytest.mark.core_model @pytest.mark.core_model
@pytest.mark.parametrize("model", models) @pytest.mark.parametrize("model", models)
@pytest.mark.parametrize(
"size_factors",
[
# Single-scale
[0.5],
# Single-scale, batched
[0.5, 0.5],
# Multi-scale
[0.25, 0.5, 0.5],
],
)
@pytest.mark.parametrize("dtype", [target_dtype]) @pytest.mark.parametrize("dtype", [target_dtype])
@pytest.mark.parametrize("max_tokens", [128]) @pytest.mark.parametrize("max_tokens", [128])
@pytest.mark.parametrize("num_logprobs", [10]) @pytest.mark.parametrize("num_logprobs", [10])
@@ -341,7 +340,6 @@ def test_qwen2_vl_image_embeddings_input(
vllm_runner, vllm_runner,
image_assets, image_assets,
model, model,
size_factors,
dtype, dtype,
max_tokens, max_tokens,
num_logprobs, num_logprobs,
@@ -355,6 +353,7 @@ def test_qwen2_vl_image_embeddings_input(
[rescale_image_size(image, factor) for factor in size_factors], [rescale_image_size(image, factor) for factor in size_factors],
[], [],
) )
for size_factors in IMAGE_SIZE_FACTOR_GROUPS
for image, prompt in zip(images, IMAGE_PROMPTS) for image, prompt in zip(images, IMAGE_PROMPTS)
] ]
@@ -372,17 +371,6 @@ def test_qwen2_vl_image_embeddings_input(
@pytest.mark.core_model @pytest.mark.core_model
@pytest.mark.parametrize("model", models) @pytest.mark.parametrize("model", models)
@pytest.mark.parametrize(
"size_factors",
[
# Single-scale
[0.5],
# Single-scale, batched
[0.5, 0.5],
# Multi-scale
[0.25, 0.5, 0.5],
],
)
@pytest.mark.parametrize("dtype", [target_dtype]) @pytest.mark.parametrize("dtype", [target_dtype])
@pytest.mark.parametrize("max_tokens", [128]) @pytest.mark.parametrize("max_tokens", [128])
@pytest.mark.parametrize("num_logprobs", [10]) @pytest.mark.parametrize("num_logprobs", [10])
@@ -390,7 +378,6 @@ def test_qwen2_vl_multiple_image_embeddings_input(
vllm_runner, vllm_runner,
image_assets, image_assets,
model, model,
size_factors,
dtype: str, dtype: str,
max_tokens: int, max_tokens: int,
num_logprobs: int, num_logprobs: int,
@@ -406,6 +393,7 @@ def test_qwen2_vl_multiple_image_embeddings_input(
], ],
[], [],
) )
for size_factors in IMAGE_SIZE_FACTOR_GROUPS
] ]
run_embedding_input_test( run_embedding_input_test(
@@ -422,17 +410,6 @@ def test_qwen2_vl_multiple_image_embeddings_input(
@pytest.mark.core_model @pytest.mark.core_model
@pytest.mark.parametrize("model", models) @pytest.mark.parametrize("model", models)
@pytest.mark.parametrize(
"size_factors",
[
# Single-scale
[0.5],
# Single-scale, batched
[0.5, 0.5],
# Multi-scale
[0.25, 0.25, 0.5],
],
)
@pytest.mark.parametrize("dtype", [target_dtype]) @pytest.mark.parametrize("dtype", [target_dtype])
@pytest.mark.parametrize("max_tokens", [128]) @pytest.mark.parametrize("max_tokens", [128])
@pytest.mark.parametrize("num_logprobs", [10]) @pytest.mark.parametrize("num_logprobs", [10])
@@ -440,7 +417,6 @@ def test_qwen2_vl_video_embeddings_input(
vllm_runner, vllm_runner,
video_assets, video_assets,
model, model,
size_factors,
dtype: str, dtype: str,
max_tokens: int, max_tokens: int,
num_logprobs: int, num_logprobs: int,
@@ -457,6 +433,7 @@ def test_qwen2_vl_video_embeddings_input(
[], [],
[rescale_video_size(video, factor) for factor in size_factors], [rescale_video_size(video, factor) for factor in size_factors],
) )
for size_factors in VIDEO_SIZE_FACTOR_GROUPS
for video, prompt in zip(sampled_vids, VIDEO_PROMPTS) for video, prompt in zip(sampled_vids, VIDEO_PROMPTS)
] ]
@@ -94,7 +94,8 @@ def get_model_type_cases(
test_info.needs_video_metadata test_info.needs_video_metadata
) )
# No sizes passed for custom inputs, since inputs are directly provided # Keep all size batches in one test case so they share the same model
# instances. No sizes are passed for preprocessed audio or custom inputs.
if test_type not in ( if test_type not in (
VLMTestType.CUSTOM_INPUTS, VLMTestType.CUSTOM_INPUTS,
VLMTestType.AUDIO, VLMTestType.AUDIO,
@@ -102,7 +103,9 @@ def get_model_type_cases(
wrapped_sizes = get_wrapped_test_sizes(test_info, test_type) wrapped_sizes = get_wrapped_test_sizes(test_info, test_type)
if wrapped_sizes is None: if wrapped_sizes is None:
raise ValueError(f"Sizes must be set for test type {test_type}") raise ValueError(f"Sizes must be set for test type {test_type}")
iter_kwargs["size_wrapper"] = wrapped_sizes if not wrapped_sizes:
return []
iter_kwargs["size_wrappers"] = (wrapped_sizes,)
# Otherwise expand the custom test options instead # Otherwise expand the custom test options instead
elif test_type == VLMTestType.CUSTOM_INPUTS: elif test_type == VLMTestType.CUSTOM_INPUTS:
@@ -127,9 +130,8 @@ def get_parametrized_options(
create_new_process_for_each_test: bool, create_new_process_for_each_test: bool,
): ):
"""Converts all of our VLMTestInfo into an expanded list of parameters. """Converts all of our VLMTestInfo into an expanded list of parameters.
This is similar to nesting pytest parametrize calls, but done directly Runner configuration values are expanded through an itertools product.
through an itertools product so that each test can set things like Input size batches stay grouped so they can share model instances.
size factors etc, while still running in isolated test cases.
""" """
matching_tests = get_filtered_test_settings( matching_tests = get_filtered_test_settings(
test_settings, test_type, create_new_process_for_each_test test_settings, test_type, create_new_process_for_each_test
@@ -149,8 +151,7 @@ def get_wrapped_test_sizes(
test_info: VLMTestInfo, test_type: VLMTestType test_info: VLMTestInfo, test_type: VLMTestType
) -> tuple[ImageSizeWrapper, ...]: ) -> tuple[ImageSizeWrapper, ...]:
"""Given a test info which may have size factors or fixed sizes, wrap them """Given a test info which may have size factors or fixed sizes, wrap them
and combine them into an iterable, each of which will be used in parameter and combine them into an iterable of request batches.
expansion.
Args: Args:
test_info: Test configuration to be expanded. test_info: Test configuration to be expanded.
@@ -4,6 +4,7 @@
types / modalities. types / modalities.
""" """
import itertools
from pathlib import PosixPath from pathlib import PosixPath
from .....conftest import ( from .....conftest import (
@@ -27,9 +28,14 @@ def run_single_image_test(
vllm_runner: type[VllmRunner], vllm_runner: type[VllmRunner],
image_assets: ImageTestAssets, image_assets: ImageTestAssets,
): ):
assert test_case.size_wrapper is not None assert test_case.size_wrappers
inputs = builders.build_single_image_inputs_from_test_info( inputs = list(
model_test_info, image_assets, test_case.size_wrapper, tmp_path itertools.chain.from_iterable(
builders.build_single_image_inputs_from_test_info(
model_test_info, image_assets, size_wrapper, tmp_path
)
for size_wrapper in test_case.size_wrappers
)
) )
core.run_test( core.run_test(
@@ -55,9 +61,14 @@ def run_multi_image_test(
vllm_runner: type[VllmRunner], vllm_runner: type[VllmRunner],
image_assets: ImageTestAssets, image_assets: ImageTestAssets,
): ):
assert test_case.size_wrapper is not None assert test_case.size_wrappers
inputs = builders.build_multi_image_inputs_from_test_info( inputs = list(
model_test_info, image_assets, test_case.size_wrapper, tmp_path itertools.chain.from_iterable(
builders.build_multi_image_inputs_from_test_info(
model_test_info, image_assets, size_wrapper, tmp_path
)
for size_wrapper in test_case.size_wrappers
)
) )
core.run_test( core.run_test(
@@ -82,9 +93,20 @@ def run_embedding_test(
vllm_runner: type[VllmRunner], vllm_runner: type[VllmRunner],
image_assets: ImageTestAssets, image_assets: ImageTestAssets,
): ):
assert test_case.size_wrapper is not None assert test_case.size_wrappers
inputs, vllm_embeddings = builders.build_embedding_inputs_from_test_info( inputs_and_embeddings = [
model_test_info, image_assets, test_case.size_wrapper builders.build_embedding_inputs_from_test_info(
model_test_info, image_assets, size_wrapper
)
for size_wrapper in test_case.size_wrappers
]
inputs = list(
itertools.chain.from_iterable(inputs for inputs, _ in inputs_and_embeddings)
)
vllm_embeddings = list(
itertools.chain.from_iterable(
embeddings for _, embeddings in inputs_and_embeddings
)
) )
core.run_test( core.run_test(
@@ -110,14 +132,19 @@ def run_video_test(
vllm_runner: type[VllmRunner], vllm_runner: type[VllmRunner],
video_assets: VideoTestAssets, video_assets: VideoTestAssets,
): ):
assert test_case.size_wrapper is not None assert test_case.size_wrappers
assert test_case.num_video_frames is not None assert test_case.num_video_frames is not None
inputs = builders.build_video_inputs_from_test_info( inputs = list(
model_test_info, itertools.chain.from_iterable(
video_assets, builders.build_video_inputs_from_test_info(
test_case.size_wrapper, model_test_info,
test_case.num_video_frames, video_assets,
test_case.needs_video_metadata, size_wrapper,
test_case.num_video_frames,
test_case.needs_video_metadata,
)
for size_wrapper in test_case.size_wrappers
)
) )
core.run_test( core.run_test(
@@ -158,11 +158,9 @@ class VLMTestInfo(NamedTuple):
num_video_frames: int | tuple[int] = 16 num_video_frames: int | tuple[int] = 16
needs_video_metadata: bool = False needs_video_metadata: bool = False
# Fixed image sizes / image size factors; most tests use image_size_factors # Fixed image sizes / image size factors; most tests use image_size_factors.
# The values provided for these two fields will be stacked and expanded # Each inner iterable defines one request batch. All batches are run against
# such that each model will consider each image size factor / image size # the same model instance.
# once per tests (much like concatenating and wrapping in one parametrize
# call)
image_size_factors: Iterable[Iterable[float]] = IMAGE_SIZE_FACTORS image_size_factors: Iterable[Iterable[float]] = IMAGE_SIZE_FACTORS
image_sizes: Iterable[Iterable[tuple[int, int]]] | None = None image_sizes: Iterable[Iterable[tuple[int, int]]] | None = None
@@ -211,8 +209,8 @@ class ExpandableVLMTestArgs(NamedTuple):
num_logprobs: int num_logprobs: int
dtype: str dtype: str
distributed_executor_backend: str | None distributed_executor_backend: str | None
# Sizes are used for everything except for custom input tests # Sizes are used for everything except audio and custom input tests.
size_wrapper: ImageSizeWrapper | None = None size_wrappers: tuple[ImageSizeWrapper, ...] = ()
# Video only # Video only
num_video_frames: int | None = None num_video_frames: int | None = None
needs_video_metadata: bool = False needs_video_metadata: bool = False