iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[CI/Build] Avoid duplicate runner startup for multimodal test (#52417)

Browse Source 
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-08-16 22:17:25 +00:00
committed by GitHub
parent 6b0b850a8b
commit e3c1cb54fc
5 changed files with 77 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/models/multimodal/generation/test_phi4mm.py
+8 -27
View File
@@ -66,6 +66,11 @@ def vllm_to_hf_output(
target_dtype = "half"
IMAGE_SIZE_FACTOR_GROUPS = (
(1.0,),
(1.0, 1.0, 1.0),
(0.25, 0.5, 1.0),
)
def run_test(
@@ -167,17 +172,6 @@ def run_test(
@pytest.mark.parametrize("model", models)
@pytest.mark.parametrize(
"size_factors",
[
# Single-scale
[1.0],
# Single-scale, batched
[1.0, 1.0, 1.0],
# Multi-scale
[0.25, 0.5, 1.0],
],
)
@pytest.mark.parametrize("dtype", [target_dtype])
@pytest.mark.parametrize("max_model_len", [12800])
@pytest.mark.parametrize("max_tokens", [128])
@@ -187,7 +181,6 @@ def test_models(
vllm_runner,
image_assets,
model,
size_factors,
dtype: str,
max_model_len: int,
max_tokens: int,
@@ -201,6 +194,7 @@ def test_models(
[rescale_image_size(image, factor) for factor in size_factors],
None,
)
for size_factors in IMAGE_SIZE_FACTOR_GROUPS
for image, prompt in zip(images, HF_IMAGE_PROMPTS)
]
@@ -220,19 +214,6 @@ def test_models(
@large_gpu_test(min_gb=48)
@pytest.mark.parametrize("model", models)
@pytest.mark.parametrize(
"size_factors",
[
# No image
# [],
# Single-scale
[1.0],
# Single-scale, batched
[1.0, 1.0, 1.0],
# Multi-scale
[0.25, 0.5, 1.0],
],
)
@pytest.mark.parametrize("dtype", [target_dtype])
@pytest.mark.parametrize("max_model_len", [25600])
@pytest.mark.parametrize("max_tokens", [128])
@@ -242,7 +223,6 @@ def test_multi_images_models(
vllm_runner,
image_assets,
model,
size_factors,
dtype: str,
max_model_len: int,
max_tokens: int,
@@ -258,7 +238,8 @@ def test_multi_images_models(
for factor in size_factors
],
None,
),
)
for size_factors in IMAGE_SIZE_FACTOR_GROUPS
]
run_test(
tests/models/multimodal/generation/test_qwen2_vl.py
+13 -36
View File
@@ -29,6 +29,16 @@ def enable_pickle(monkeypatch):
models = ["Qwen/Qwen2-VL-2B-Instruct"]
target_dtype = "half"
IMAGE_SIZE_FACTOR_GROUPS = (
(0.5,),
(0.5, 0.5),
(0.25, 0.5, 0.5),
)
VIDEO_SIZE_FACTOR_GROUPS = (
(0.5,),
(0.5, 0.5),
(0.25, 0.25, 0.5),
)
IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>"
VIDEO_PLACEHOLDER = "<|vision_start|><|video_pad|><|vision_end|>"
@@ -323,17 +333,6 @@ def run_embedding_input_test(
@pytest.mark.core_model
@pytest.mark.parametrize("model", models)
@pytest.mark.parametrize(
"size_factors",
[
# Single-scale
[0.5],
# Single-scale, batched
[0.5, 0.5],
# Multi-scale
[0.25, 0.5, 0.5],
],
)
@pytest.mark.parametrize("dtype", [target_dtype])
@pytest.mark.parametrize("max_tokens", [128])
@pytest.mark.parametrize("num_logprobs", [10])
@@ -341,7 +340,6 @@ def test_qwen2_vl_image_embeddings_input(
vllm_runner,
image_assets,
model,
size_factors,
dtype,
max_tokens,
num_logprobs,
@@ -355,6 +353,7 @@ def test_qwen2_vl_image_embeddings_input(
[rescale_image_size(image, factor) for factor in size_factors],
[],
)
for size_factors in IMAGE_SIZE_FACTOR_GROUPS
for image, prompt in zip(images, IMAGE_PROMPTS)
]
@@ -372,17 +371,6 @@ def test_qwen2_vl_image_embeddings_input(
@pytest.mark.core_model
@pytest.mark.parametrize("model", models)
@pytest.mark.parametrize(
"size_factors",
[
# Single-scale
[0.5],
# Single-scale, batched
[0.5, 0.5],
# Multi-scale
[0.25, 0.5, 0.5],
],
)
@pytest.mark.parametrize("dtype", [target_dtype])
@pytest.mark.parametrize("max_tokens", [128])
@pytest.mark.parametrize("num_logprobs", [10])
@@ -390,7 +378,6 @@ def test_qwen2_vl_multiple_image_embeddings_input(
vllm_runner,
image_assets,
model,
size_factors,
dtype: str,
max_tokens: int,
num_logprobs: int,
@@ -406,6 +393,7 @@ def test_qwen2_vl_multiple_image_embeddings_input(
],
[],
)
for size_factors in IMAGE_SIZE_FACTOR_GROUPS
]
run_embedding_input_test(
@@ -422,17 +410,6 @@ def test_qwen2_vl_multiple_image_embeddings_input(
@pytest.mark.core_model
@pytest.mark.parametrize("model", models)
@pytest.mark.parametrize(
"size_factors",
[
# Single-scale
[0.5],
# Single-scale, batched
[0.5, 0.5],
# Multi-scale
[0.25, 0.25, 0.5],
],
)
@pytest.mark.parametrize("dtype", [target_dtype])
@pytest.mark.parametrize("max_tokens", [128])
@pytest.mark.parametrize("num_logprobs", [10])
@@ -440,7 +417,6 @@ def test_qwen2_vl_video_embeddings_input(
vllm_runner,
video_assets,
model,
size_factors,
dtype: str,
max_tokens: int,
num_logprobs: int,
@@ -457,6 +433,7 @@ def test_qwen2_vl_video_embeddings_input(
[],
[rescale_video_size(video, factor) for factor in size_factors],
)
for size_factors in VIDEO_SIZE_FACTOR_GROUPS
for video, prompt in zip(sampled_vids, VIDEO_PROMPTS)
]
tests/models/multimodal/generation/vlm_utils/case_filtering.py
+8 -7
View File
@@ -94,7 +94,8 @@ def get_model_type_cases(
test_info.needs_video_metadata
)
# No sizes passed for custom inputs, since inputs are directly provided
# Keep all size batches in one test case so they share the same model
# instances. No sizes are passed for preprocessed audio or custom inputs.
if test_type not in (
VLMTestType.CUSTOM_INPUTS,
VLMTestType.AUDIO,
@@ -102,7 +103,9 @@ def get_model_type_cases(
wrapped_sizes = get_wrapped_test_sizes(test_info, test_type)
if wrapped_sizes is None:
raise ValueError(f"Sizes must be set for test type {test_type}")
iter_kwargs["size_wrapper"] = wrapped_sizes
if not wrapped_sizes:
return []
iter_kwargs["size_wrappers"] = (wrapped_sizes,)
# Otherwise expand the custom test options instead
elif test_type == VLMTestType.CUSTOM_INPUTS:
@@ -127,9 +130,8 @@ def get_parametrized_options(
create_new_process_for_each_test: bool,
):
"""Converts all of our VLMTestInfo into an expanded list of parameters.
This is similar to nesting pytest parametrize calls, but done directly
through an itertools product so that each test can set things like
size factors etc, while still running in isolated test cases.
Runner configuration values are expanded through an itertools product.
Input size batches stay grouped so they can share model instances.
"""
matching_tests = get_filtered_test_settings(
test_settings, test_type, create_new_process_for_each_test
@@ -149,8 +151,7 @@ def get_wrapped_test_sizes(
test_info: VLMTestInfo, test_type: VLMTestType
) -> tuple[ImageSizeWrapper, ...]:
"""Given a test info which may have size factors or fixed sizes, wrap them
and combine them into an iterable, each of which will be used in parameter
expansion.
and combine them into an iterable of request batches.
Args:
test_info: Test configuration to be expanded.
tests/models/multimodal/generation/vlm_utils/runners.py
+43 -16
View File
@@ -4,6 +4,7 @@
types / modalities.
"""
import itertools
from pathlib import PosixPath
from .....conftest import (
@@ -27,9 +28,14 @@ def run_single_image_test(
vllm_runner: type[VllmRunner],
image_assets: ImageTestAssets,
):
assert test_case.size_wrapper is not None
inputs = builders.build_single_image_inputs_from_test_info(
model_test_info, image_assets, test_case.size_wrapper, tmp_path
assert test_case.size_wrappers
inputs = list(
itertools.chain.from_iterable(
builders.build_single_image_inputs_from_test_info(
model_test_info, image_assets, size_wrapper, tmp_path
)
for size_wrapper in test_case.size_wrappers
)
)
core.run_test(
@@ -55,9 +61,14 @@ def run_multi_image_test(
vllm_runner: type[VllmRunner],
image_assets: ImageTestAssets,
):
assert test_case.size_wrapper is not None
inputs = builders.build_multi_image_inputs_from_test_info(
model_test_info, image_assets, test_case.size_wrapper, tmp_path
assert test_case.size_wrappers
inputs = list(
itertools.chain.from_iterable(
builders.build_multi_image_inputs_from_test_info(
model_test_info, image_assets, size_wrapper, tmp_path
)
for size_wrapper in test_case.size_wrappers
)
)
core.run_test(
@@ -82,9 +93,20 @@ def run_embedding_test(
vllm_runner: type[VllmRunner],
image_assets: ImageTestAssets,
):
assert test_case.size_wrapper is not None
inputs, vllm_embeddings = builders.build_embedding_inputs_from_test_info(
model_test_info, image_assets, test_case.size_wrapper
assert test_case.size_wrappers
inputs_and_embeddings = [
builders.build_embedding_inputs_from_test_info(
model_test_info, image_assets, size_wrapper
)
for size_wrapper in test_case.size_wrappers
]
inputs = list(
itertools.chain.from_iterable(inputs for inputs, _ in inputs_and_embeddings)
)
vllm_embeddings = list(
itertools.chain.from_iterable(
embeddings for _, embeddings in inputs_and_embeddings
)
)
core.run_test(
@@ -110,14 +132,19 @@ def run_video_test(
vllm_runner: type[VllmRunner],
video_assets: VideoTestAssets,
):
assert test_case.size_wrapper is not None
assert test_case.size_wrappers
assert test_case.num_video_frames is not None
inputs = builders.build_video_inputs_from_test_info(
model_test_info,
video_assets,
test_case.size_wrapper,
test_case.num_video_frames,
test_case.needs_video_metadata,
inputs = list(
itertools.chain.from_iterable(
builders.build_video_inputs_from_test_info(
model_test_info,
video_assets,
size_wrapper,
test_case.num_video_frames,
test_case.needs_video_metadata,
)
for size_wrapper in test_case.size_wrappers
)
)
core.run_test(
tests/models/multimodal/generation/vlm_utils/types.py
+5 -7
View File
@@ -158,11 +158,9 @@ class VLMTestInfo(NamedTuple):
num_video_frames: int | tuple[int] = 16
needs_video_metadata: bool = False
# Fixed image sizes / image size factors; most tests use image_size_factors
# The values provided for these two fields will be stacked and expanded
# such that each model will consider each image size factor / image size
# once per tests (much like concatenating and wrapping in one parametrize
# call)
# Fixed image sizes / image size factors; most tests use image_size_factors.
# Each inner iterable defines one request batch. All batches are run against
# the same model instance.
image_size_factors: Iterable[Iterable[float]] = IMAGE_SIZE_FACTORS
image_sizes: Iterable[Iterable[tuple[int, int]]] | None = None
@@ -211,8 +209,8 @@ class ExpandableVLMTestArgs(NamedTuple):
num_logprobs: int
dtype: str
distributed_executor_backend: str | None
# Sizes are used for everything except for custom input tests
size_wrapper: ImageSizeWrapper | None = None
# Sizes are used for everything except audio and custom input tests.
size_wrappers: tuple[ImageSizeWrapper, ...] = ()
# Video only
num_video_frames: int | None = None
needs_video_metadata: bool = False