From e3c1cb54fcafe796bc2a0354bb0962edbb4abbb8 Mon Sep 17 00:00:00 2001 From: Isotr0py Date: Mon, 17 Aug 2026 06:17:25 +0800 Subject: [PATCH] [CI/Build] Avoid duplicate runner startup for multimodal test (#52417) Signed-off-by: Isotr0py --- .../multimodal/generation/test_phi4mm.py | 35 +++-------- .../multimodal/generation/test_qwen2_vl.py | 49 ++++----------- .../generation/vlm_utils/case_filtering.py | 15 ++--- .../generation/vlm_utils/runners.py | 59 ++++++++++++++----- .../multimodal/generation/vlm_utils/types.py | 12 ++-- 5 files changed, 77 insertions(+), 93 deletions(-) diff --git a/tests/models/multimodal/generation/test_phi4mm.py b/tests/models/multimodal/generation/test_phi4mm.py index 5ab75e145a..71dbfcd5f3 100644 --- a/tests/models/multimodal/generation/test_phi4mm.py +++ b/tests/models/multimodal/generation/test_phi4mm.py @@ -66,6 +66,11 @@ def vllm_to_hf_output( target_dtype = "half" +IMAGE_SIZE_FACTOR_GROUPS = ( + (1.0,), + (1.0, 1.0, 1.0), + (0.25, 0.5, 1.0), +) def run_test( @@ -167,17 +172,6 @@ def run_test( @pytest.mark.parametrize("model", models) -@pytest.mark.parametrize( - "size_factors", - [ - # Single-scale - [1.0], - # Single-scale, batched - [1.0, 1.0, 1.0], - # Multi-scale - [0.25, 0.5, 1.0], - ], -) @pytest.mark.parametrize("dtype", [target_dtype]) @pytest.mark.parametrize("max_model_len", [12800]) @pytest.mark.parametrize("max_tokens", [128]) @@ -187,7 +181,6 @@ def test_models( vllm_runner, image_assets, model, - size_factors, dtype: str, max_model_len: int, max_tokens: int, @@ -201,6 +194,7 @@ def test_models( [rescale_image_size(image, factor) for factor in size_factors], None, ) + for size_factors in IMAGE_SIZE_FACTOR_GROUPS for image, prompt in zip(images, HF_IMAGE_PROMPTS) ] @@ -220,19 +214,6 @@ def test_models( @large_gpu_test(min_gb=48) @pytest.mark.parametrize("model", models) -@pytest.mark.parametrize( - "size_factors", - [ - # No image - # [], - # Single-scale - [1.0], - # Single-scale, batched - [1.0, 1.0, 1.0], - # Multi-scale - [0.25, 0.5, 1.0], - ], -) @pytest.mark.parametrize("dtype", [target_dtype]) @pytest.mark.parametrize("max_model_len", [25600]) @pytest.mark.parametrize("max_tokens", [128]) @@ -242,7 +223,6 @@ def test_multi_images_models( vllm_runner, image_assets, model, - size_factors, dtype: str, max_model_len: int, max_tokens: int, @@ -258,7 +238,8 @@ def test_multi_images_models( for factor in size_factors ], None, - ), + ) + for size_factors in IMAGE_SIZE_FACTOR_GROUPS ] run_test( diff --git a/tests/models/multimodal/generation/test_qwen2_vl.py b/tests/models/multimodal/generation/test_qwen2_vl.py index 6148c0bcda..19328cad7a 100644 --- a/tests/models/multimodal/generation/test_qwen2_vl.py +++ b/tests/models/multimodal/generation/test_qwen2_vl.py @@ -29,6 +29,16 @@ def enable_pickle(monkeypatch): models = ["Qwen/Qwen2-VL-2B-Instruct"] target_dtype = "half" +IMAGE_SIZE_FACTOR_GROUPS = ( + (0.5,), + (0.5, 0.5), + (0.25, 0.5, 0.5), +) +VIDEO_SIZE_FACTOR_GROUPS = ( + (0.5,), + (0.5, 0.5), + (0.25, 0.25, 0.5), +) IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>" VIDEO_PLACEHOLDER = "<|vision_start|><|video_pad|><|vision_end|>" @@ -323,17 +333,6 @@ def run_embedding_input_test( @pytest.mark.core_model @pytest.mark.parametrize("model", models) -@pytest.mark.parametrize( - "size_factors", - [ - # Single-scale - [0.5], - # Single-scale, batched - [0.5, 0.5], - # Multi-scale - [0.25, 0.5, 0.5], - ], -) @pytest.mark.parametrize("dtype", [target_dtype]) @pytest.mark.parametrize("max_tokens", [128]) @pytest.mark.parametrize("num_logprobs", [10]) @@ -341,7 +340,6 @@ def test_qwen2_vl_image_embeddings_input( vllm_runner, image_assets, model, - size_factors, dtype, max_tokens, num_logprobs, @@ -355,6 +353,7 @@ def test_qwen2_vl_image_embeddings_input( [rescale_image_size(image, factor) for factor in size_factors], [], ) + for size_factors in IMAGE_SIZE_FACTOR_GROUPS for image, prompt in zip(images, IMAGE_PROMPTS) ] @@ -372,17 +371,6 @@ def test_qwen2_vl_image_embeddings_input( @pytest.mark.core_model @pytest.mark.parametrize("model", models) -@pytest.mark.parametrize( - "size_factors", - [ - # Single-scale - [0.5], - # Single-scale, batched - [0.5, 0.5], - # Multi-scale - [0.25, 0.5, 0.5], - ], -) @pytest.mark.parametrize("dtype", [target_dtype]) @pytest.mark.parametrize("max_tokens", [128]) @pytest.mark.parametrize("num_logprobs", [10]) @@ -390,7 +378,6 @@ def test_qwen2_vl_multiple_image_embeddings_input( vllm_runner, image_assets, model, - size_factors, dtype: str, max_tokens: int, num_logprobs: int, @@ -406,6 +393,7 @@ def test_qwen2_vl_multiple_image_embeddings_input( ], [], ) + for size_factors in IMAGE_SIZE_FACTOR_GROUPS ] run_embedding_input_test( @@ -422,17 +410,6 @@ def test_qwen2_vl_multiple_image_embeddings_input( @pytest.mark.core_model @pytest.mark.parametrize("model", models) -@pytest.mark.parametrize( - "size_factors", - [ - # Single-scale - [0.5], - # Single-scale, batched - [0.5, 0.5], - # Multi-scale - [0.25, 0.25, 0.5], - ], -) @pytest.mark.parametrize("dtype", [target_dtype]) @pytest.mark.parametrize("max_tokens", [128]) @pytest.mark.parametrize("num_logprobs", [10]) @@ -440,7 +417,6 @@ def test_qwen2_vl_video_embeddings_input( vllm_runner, video_assets, model, - size_factors, dtype: str, max_tokens: int, num_logprobs: int, @@ -457,6 +433,7 @@ def test_qwen2_vl_video_embeddings_input( [], [rescale_video_size(video, factor) for factor in size_factors], ) + for size_factors in VIDEO_SIZE_FACTOR_GROUPS for video, prompt in zip(sampled_vids, VIDEO_PROMPTS) ] diff --git a/tests/models/multimodal/generation/vlm_utils/case_filtering.py b/tests/models/multimodal/generation/vlm_utils/case_filtering.py index 116eead7a7..cbe6604257 100644 --- a/tests/models/multimodal/generation/vlm_utils/case_filtering.py +++ b/tests/models/multimodal/generation/vlm_utils/case_filtering.py @@ -94,7 +94,8 @@ def get_model_type_cases( test_info.needs_video_metadata ) - # No sizes passed for custom inputs, since inputs are directly provided + # Keep all size batches in one test case so they share the same model + # instances. No sizes are passed for preprocessed audio or custom inputs. if test_type not in ( VLMTestType.CUSTOM_INPUTS, VLMTestType.AUDIO, @@ -102,7 +103,9 @@ def get_model_type_cases( wrapped_sizes = get_wrapped_test_sizes(test_info, test_type) if wrapped_sizes is None: raise ValueError(f"Sizes must be set for test type {test_type}") - iter_kwargs["size_wrapper"] = wrapped_sizes + if not wrapped_sizes: + return [] + iter_kwargs["size_wrappers"] = (wrapped_sizes,) # Otherwise expand the custom test options instead elif test_type == VLMTestType.CUSTOM_INPUTS: @@ -127,9 +130,8 @@ def get_parametrized_options( create_new_process_for_each_test: bool, ): """Converts all of our VLMTestInfo into an expanded list of parameters. - This is similar to nesting pytest parametrize calls, but done directly - through an itertools product so that each test can set things like - size factors etc, while still running in isolated test cases. + Runner configuration values are expanded through an itertools product. + Input size batches stay grouped so they can share model instances. """ matching_tests = get_filtered_test_settings( test_settings, test_type, create_new_process_for_each_test @@ -149,8 +151,7 @@ def get_wrapped_test_sizes( test_info: VLMTestInfo, test_type: VLMTestType ) -> tuple[ImageSizeWrapper, ...]: """Given a test info which may have size factors or fixed sizes, wrap them - and combine them into an iterable, each of which will be used in parameter - expansion. + and combine them into an iterable of request batches. Args: test_info: Test configuration to be expanded. diff --git a/tests/models/multimodal/generation/vlm_utils/runners.py b/tests/models/multimodal/generation/vlm_utils/runners.py index 218339ef1d..571ea54250 100644 --- a/tests/models/multimodal/generation/vlm_utils/runners.py +++ b/tests/models/multimodal/generation/vlm_utils/runners.py @@ -4,6 +4,7 @@ types / modalities. """ +import itertools from pathlib import PosixPath from .....conftest import ( @@ -27,9 +28,14 @@ def run_single_image_test( vllm_runner: type[VllmRunner], image_assets: ImageTestAssets, ): - assert test_case.size_wrapper is not None - inputs = builders.build_single_image_inputs_from_test_info( - model_test_info, image_assets, test_case.size_wrapper, tmp_path + assert test_case.size_wrappers + inputs = list( + itertools.chain.from_iterable( + builders.build_single_image_inputs_from_test_info( + model_test_info, image_assets, size_wrapper, tmp_path + ) + for size_wrapper in test_case.size_wrappers + ) ) core.run_test( @@ -55,9 +61,14 @@ def run_multi_image_test( vllm_runner: type[VllmRunner], image_assets: ImageTestAssets, ): - assert test_case.size_wrapper is not None - inputs = builders.build_multi_image_inputs_from_test_info( - model_test_info, image_assets, test_case.size_wrapper, tmp_path + assert test_case.size_wrappers + inputs = list( + itertools.chain.from_iterable( + builders.build_multi_image_inputs_from_test_info( + model_test_info, image_assets, size_wrapper, tmp_path + ) + for size_wrapper in test_case.size_wrappers + ) ) core.run_test( @@ -82,9 +93,20 @@ def run_embedding_test( vllm_runner: type[VllmRunner], image_assets: ImageTestAssets, ): - assert test_case.size_wrapper is not None - inputs, vllm_embeddings = builders.build_embedding_inputs_from_test_info( - model_test_info, image_assets, test_case.size_wrapper + assert test_case.size_wrappers + inputs_and_embeddings = [ + builders.build_embedding_inputs_from_test_info( + model_test_info, image_assets, size_wrapper + ) + for size_wrapper in test_case.size_wrappers + ] + inputs = list( + itertools.chain.from_iterable(inputs for inputs, _ in inputs_and_embeddings) + ) + vllm_embeddings = list( + itertools.chain.from_iterable( + embeddings for _, embeddings in inputs_and_embeddings + ) ) core.run_test( @@ -110,14 +132,19 @@ def run_video_test( vllm_runner: type[VllmRunner], video_assets: VideoTestAssets, ): - assert test_case.size_wrapper is not None + assert test_case.size_wrappers assert test_case.num_video_frames is not None - inputs = builders.build_video_inputs_from_test_info( - model_test_info, - video_assets, - test_case.size_wrapper, - test_case.num_video_frames, - test_case.needs_video_metadata, + inputs = list( + itertools.chain.from_iterable( + builders.build_video_inputs_from_test_info( + model_test_info, + video_assets, + size_wrapper, + test_case.num_video_frames, + test_case.needs_video_metadata, + ) + for size_wrapper in test_case.size_wrappers + ) ) core.run_test( diff --git a/tests/models/multimodal/generation/vlm_utils/types.py b/tests/models/multimodal/generation/vlm_utils/types.py index af48a1479b..3722caed2c 100644 --- a/tests/models/multimodal/generation/vlm_utils/types.py +++ b/tests/models/multimodal/generation/vlm_utils/types.py @@ -158,11 +158,9 @@ class VLMTestInfo(NamedTuple): num_video_frames: int | tuple[int] = 16 needs_video_metadata: bool = False - # Fixed image sizes / image size factors; most tests use image_size_factors - # The values provided for these two fields will be stacked and expanded - # such that each model will consider each image size factor / image size - # once per tests (much like concatenating and wrapping in one parametrize - # call) + # Fixed image sizes / image size factors; most tests use image_size_factors. + # Each inner iterable defines one request batch. All batches are run against + # the same model instance. image_size_factors: Iterable[Iterable[float]] = IMAGE_SIZE_FACTORS image_sizes: Iterable[Iterable[tuple[int, int]]] | None = None @@ -211,8 +209,8 @@ class ExpandableVLMTestArgs(NamedTuple): num_logprobs: int dtype: str distributed_executor_backend: str | None - # Sizes are used for everything except for custom input tests - size_wrapper: ImageSizeWrapper | None = None + # Sizes are used for everything except audio and custom input tests. + size_wrappers: tuple[ImageSizeWrapper, ...] = () # Video only num_video_frames: int | None = None needs_video_metadata: bool = False