[CI/Build] Avoid duplicate runner startup for multimodal test (#52417)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -66,6 +66,11 @@ def vllm_to_hf_output(
|
|||||||
|
|
||||||
|
|
||||||
target_dtype = "half"
|
target_dtype = "half"
|
||||||
|
IMAGE_SIZE_FACTOR_GROUPS = (
|
||||||
|
(1.0,),
|
||||||
|
(1.0, 1.0, 1.0),
|
||||||
|
(0.25, 0.5, 1.0),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def run_test(
|
def run_test(
|
||||||
@@ -167,17 +172,6 @@ def run_test(
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("model", models)
|
@pytest.mark.parametrize("model", models)
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"size_factors",
|
|
||||||
[
|
|
||||||
# Single-scale
|
|
||||||
[1.0],
|
|
||||||
# Single-scale, batched
|
|
||||||
[1.0, 1.0, 1.0],
|
|
||||||
# Multi-scale
|
|
||||||
[0.25, 0.5, 1.0],
|
|
||||||
],
|
|
||||||
)
|
|
||||||
@pytest.mark.parametrize("dtype", [target_dtype])
|
@pytest.mark.parametrize("dtype", [target_dtype])
|
||||||
@pytest.mark.parametrize("max_model_len", [12800])
|
@pytest.mark.parametrize("max_model_len", [12800])
|
||||||
@pytest.mark.parametrize("max_tokens", [128])
|
@pytest.mark.parametrize("max_tokens", [128])
|
||||||
@@ -187,7 +181,6 @@ def test_models(
|
|||||||
vllm_runner,
|
vllm_runner,
|
||||||
image_assets,
|
image_assets,
|
||||||
model,
|
model,
|
||||||
size_factors,
|
|
||||||
dtype: str,
|
dtype: str,
|
||||||
max_model_len: int,
|
max_model_len: int,
|
||||||
max_tokens: int,
|
max_tokens: int,
|
||||||
@@ -201,6 +194,7 @@ def test_models(
|
|||||||
[rescale_image_size(image, factor) for factor in size_factors],
|
[rescale_image_size(image, factor) for factor in size_factors],
|
||||||
None,
|
None,
|
||||||
)
|
)
|
||||||
|
for size_factors in IMAGE_SIZE_FACTOR_GROUPS
|
||||||
for image, prompt in zip(images, HF_IMAGE_PROMPTS)
|
for image, prompt in zip(images, HF_IMAGE_PROMPTS)
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -220,19 +214,6 @@ def test_models(
|
|||||||
|
|
||||||
@large_gpu_test(min_gb=48)
|
@large_gpu_test(min_gb=48)
|
||||||
@pytest.mark.parametrize("model", models)
|
@pytest.mark.parametrize("model", models)
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"size_factors",
|
|
||||||
[
|
|
||||||
# No image
|
|
||||||
# [],
|
|
||||||
# Single-scale
|
|
||||||
[1.0],
|
|
||||||
# Single-scale, batched
|
|
||||||
[1.0, 1.0, 1.0],
|
|
||||||
# Multi-scale
|
|
||||||
[0.25, 0.5, 1.0],
|
|
||||||
],
|
|
||||||
)
|
|
||||||
@pytest.mark.parametrize("dtype", [target_dtype])
|
@pytest.mark.parametrize("dtype", [target_dtype])
|
||||||
@pytest.mark.parametrize("max_model_len", [25600])
|
@pytest.mark.parametrize("max_model_len", [25600])
|
||||||
@pytest.mark.parametrize("max_tokens", [128])
|
@pytest.mark.parametrize("max_tokens", [128])
|
||||||
@@ -242,7 +223,6 @@ def test_multi_images_models(
|
|||||||
vllm_runner,
|
vllm_runner,
|
||||||
image_assets,
|
image_assets,
|
||||||
model,
|
model,
|
||||||
size_factors,
|
|
||||||
dtype: str,
|
dtype: str,
|
||||||
max_model_len: int,
|
max_model_len: int,
|
||||||
max_tokens: int,
|
max_tokens: int,
|
||||||
@@ -258,7 +238,8 @@ def test_multi_images_models(
|
|||||||
for factor in size_factors
|
for factor in size_factors
|
||||||
],
|
],
|
||||||
None,
|
None,
|
||||||
),
|
)
|
||||||
|
for size_factors in IMAGE_SIZE_FACTOR_GROUPS
|
||||||
]
|
]
|
||||||
|
|
||||||
run_test(
|
run_test(
|
||||||
|
|||||||
@@ -29,6 +29,16 @@ def enable_pickle(monkeypatch):
|
|||||||
|
|
||||||
models = ["Qwen/Qwen2-VL-2B-Instruct"]
|
models = ["Qwen/Qwen2-VL-2B-Instruct"]
|
||||||
target_dtype = "half"
|
target_dtype = "half"
|
||||||
|
IMAGE_SIZE_FACTOR_GROUPS = (
|
||||||
|
(0.5,),
|
||||||
|
(0.5, 0.5),
|
||||||
|
(0.25, 0.5, 0.5),
|
||||||
|
)
|
||||||
|
VIDEO_SIZE_FACTOR_GROUPS = (
|
||||||
|
(0.5,),
|
||||||
|
(0.5, 0.5),
|
||||||
|
(0.25, 0.25, 0.5),
|
||||||
|
)
|
||||||
|
|
||||||
IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>"
|
IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>"
|
||||||
VIDEO_PLACEHOLDER = "<|vision_start|><|video_pad|><|vision_end|>"
|
VIDEO_PLACEHOLDER = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||||
@@ -323,17 +333,6 @@ def run_embedding_input_test(
|
|||||||
|
|
||||||
@pytest.mark.core_model
|
@pytest.mark.core_model
|
||||||
@pytest.mark.parametrize("model", models)
|
@pytest.mark.parametrize("model", models)
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"size_factors",
|
|
||||||
[
|
|
||||||
# Single-scale
|
|
||||||
[0.5],
|
|
||||||
# Single-scale, batched
|
|
||||||
[0.5, 0.5],
|
|
||||||
# Multi-scale
|
|
||||||
[0.25, 0.5, 0.5],
|
|
||||||
],
|
|
||||||
)
|
|
||||||
@pytest.mark.parametrize("dtype", [target_dtype])
|
@pytest.mark.parametrize("dtype", [target_dtype])
|
||||||
@pytest.mark.parametrize("max_tokens", [128])
|
@pytest.mark.parametrize("max_tokens", [128])
|
||||||
@pytest.mark.parametrize("num_logprobs", [10])
|
@pytest.mark.parametrize("num_logprobs", [10])
|
||||||
@@ -341,7 +340,6 @@ def test_qwen2_vl_image_embeddings_input(
|
|||||||
vllm_runner,
|
vllm_runner,
|
||||||
image_assets,
|
image_assets,
|
||||||
model,
|
model,
|
||||||
size_factors,
|
|
||||||
dtype,
|
dtype,
|
||||||
max_tokens,
|
max_tokens,
|
||||||
num_logprobs,
|
num_logprobs,
|
||||||
@@ -355,6 +353,7 @@ def test_qwen2_vl_image_embeddings_input(
|
|||||||
[rescale_image_size(image, factor) for factor in size_factors],
|
[rescale_image_size(image, factor) for factor in size_factors],
|
||||||
[],
|
[],
|
||||||
)
|
)
|
||||||
|
for size_factors in IMAGE_SIZE_FACTOR_GROUPS
|
||||||
for image, prompt in zip(images, IMAGE_PROMPTS)
|
for image, prompt in zip(images, IMAGE_PROMPTS)
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -372,17 +371,6 @@ def test_qwen2_vl_image_embeddings_input(
|
|||||||
|
|
||||||
@pytest.mark.core_model
|
@pytest.mark.core_model
|
||||||
@pytest.mark.parametrize("model", models)
|
@pytest.mark.parametrize("model", models)
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"size_factors",
|
|
||||||
[
|
|
||||||
# Single-scale
|
|
||||||
[0.5],
|
|
||||||
# Single-scale, batched
|
|
||||||
[0.5, 0.5],
|
|
||||||
# Multi-scale
|
|
||||||
[0.25, 0.5, 0.5],
|
|
||||||
],
|
|
||||||
)
|
|
||||||
@pytest.mark.parametrize("dtype", [target_dtype])
|
@pytest.mark.parametrize("dtype", [target_dtype])
|
||||||
@pytest.mark.parametrize("max_tokens", [128])
|
@pytest.mark.parametrize("max_tokens", [128])
|
||||||
@pytest.mark.parametrize("num_logprobs", [10])
|
@pytest.mark.parametrize("num_logprobs", [10])
|
||||||
@@ -390,7 +378,6 @@ def test_qwen2_vl_multiple_image_embeddings_input(
|
|||||||
vllm_runner,
|
vllm_runner,
|
||||||
image_assets,
|
image_assets,
|
||||||
model,
|
model,
|
||||||
size_factors,
|
|
||||||
dtype: str,
|
dtype: str,
|
||||||
max_tokens: int,
|
max_tokens: int,
|
||||||
num_logprobs: int,
|
num_logprobs: int,
|
||||||
@@ -406,6 +393,7 @@ def test_qwen2_vl_multiple_image_embeddings_input(
|
|||||||
],
|
],
|
||||||
[],
|
[],
|
||||||
)
|
)
|
||||||
|
for size_factors in IMAGE_SIZE_FACTOR_GROUPS
|
||||||
]
|
]
|
||||||
|
|
||||||
run_embedding_input_test(
|
run_embedding_input_test(
|
||||||
@@ -422,17 +410,6 @@ def test_qwen2_vl_multiple_image_embeddings_input(
|
|||||||
|
|
||||||
@pytest.mark.core_model
|
@pytest.mark.core_model
|
||||||
@pytest.mark.parametrize("model", models)
|
@pytest.mark.parametrize("model", models)
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"size_factors",
|
|
||||||
[
|
|
||||||
# Single-scale
|
|
||||||
[0.5],
|
|
||||||
# Single-scale, batched
|
|
||||||
[0.5, 0.5],
|
|
||||||
# Multi-scale
|
|
||||||
[0.25, 0.25, 0.5],
|
|
||||||
],
|
|
||||||
)
|
|
||||||
@pytest.mark.parametrize("dtype", [target_dtype])
|
@pytest.mark.parametrize("dtype", [target_dtype])
|
||||||
@pytest.mark.parametrize("max_tokens", [128])
|
@pytest.mark.parametrize("max_tokens", [128])
|
||||||
@pytest.mark.parametrize("num_logprobs", [10])
|
@pytest.mark.parametrize("num_logprobs", [10])
|
||||||
@@ -440,7 +417,6 @@ def test_qwen2_vl_video_embeddings_input(
|
|||||||
vllm_runner,
|
vllm_runner,
|
||||||
video_assets,
|
video_assets,
|
||||||
model,
|
model,
|
||||||
size_factors,
|
|
||||||
dtype: str,
|
dtype: str,
|
||||||
max_tokens: int,
|
max_tokens: int,
|
||||||
num_logprobs: int,
|
num_logprobs: int,
|
||||||
@@ -457,6 +433,7 @@ def test_qwen2_vl_video_embeddings_input(
|
|||||||
[],
|
[],
|
||||||
[rescale_video_size(video, factor) for factor in size_factors],
|
[rescale_video_size(video, factor) for factor in size_factors],
|
||||||
)
|
)
|
||||||
|
for size_factors in VIDEO_SIZE_FACTOR_GROUPS
|
||||||
for video, prompt in zip(sampled_vids, VIDEO_PROMPTS)
|
for video, prompt in zip(sampled_vids, VIDEO_PROMPTS)
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|||||||
@@ -94,7 +94,8 @@ def get_model_type_cases(
|
|||||||
test_info.needs_video_metadata
|
test_info.needs_video_metadata
|
||||||
)
|
)
|
||||||
|
|
||||||
# No sizes passed for custom inputs, since inputs are directly provided
|
# Keep all size batches in one test case so they share the same model
|
||||||
|
# instances. No sizes are passed for preprocessed audio or custom inputs.
|
||||||
if test_type not in (
|
if test_type not in (
|
||||||
VLMTestType.CUSTOM_INPUTS,
|
VLMTestType.CUSTOM_INPUTS,
|
||||||
VLMTestType.AUDIO,
|
VLMTestType.AUDIO,
|
||||||
@@ -102,7 +103,9 @@ def get_model_type_cases(
|
|||||||
wrapped_sizes = get_wrapped_test_sizes(test_info, test_type)
|
wrapped_sizes = get_wrapped_test_sizes(test_info, test_type)
|
||||||
if wrapped_sizes is None:
|
if wrapped_sizes is None:
|
||||||
raise ValueError(f"Sizes must be set for test type {test_type}")
|
raise ValueError(f"Sizes must be set for test type {test_type}")
|
||||||
iter_kwargs["size_wrapper"] = wrapped_sizes
|
if not wrapped_sizes:
|
||||||
|
return []
|
||||||
|
iter_kwargs["size_wrappers"] = (wrapped_sizes,)
|
||||||
|
|
||||||
# Otherwise expand the custom test options instead
|
# Otherwise expand the custom test options instead
|
||||||
elif test_type == VLMTestType.CUSTOM_INPUTS:
|
elif test_type == VLMTestType.CUSTOM_INPUTS:
|
||||||
@@ -127,9 +130,8 @@ def get_parametrized_options(
|
|||||||
create_new_process_for_each_test: bool,
|
create_new_process_for_each_test: bool,
|
||||||
):
|
):
|
||||||
"""Converts all of our VLMTestInfo into an expanded list of parameters.
|
"""Converts all of our VLMTestInfo into an expanded list of parameters.
|
||||||
This is similar to nesting pytest parametrize calls, but done directly
|
Runner configuration values are expanded through an itertools product.
|
||||||
through an itertools product so that each test can set things like
|
Input size batches stay grouped so they can share model instances.
|
||||||
size factors etc, while still running in isolated test cases.
|
|
||||||
"""
|
"""
|
||||||
matching_tests = get_filtered_test_settings(
|
matching_tests = get_filtered_test_settings(
|
||||||
test_settings, test_type, create_new_process_for_each_test
|
test_settings, test_type, create_new_process_for_each_test
|
||||||
@@ -149,8 +151,7 @@ def get_wrapped_test_sizes(
|
|||||||
test_info: VLMTestInfo, test_type: VLMTestType
|
test_info: VLMTestInfo, test_type: VLMTestType
|
||||||
) -> tuple[ImageSizeWrapper, ...]:
|
) -> tuple[ImageSizeWrapper, ...]:
|
||||||
"""Given a test info which may have size factors or fixed sizes, wrap them
|
"""Given a test info which may have size factors or fixed sizes, wrap them
|
||||||
and combine them into an iterable, each of which will be used in parameter
|
and combine them into an iterable of request batches.
|
||||||
expansion.
|
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
test_info: Test configuration to be expanded.
|
test_info: Test configuration to be expanded.
|
||||||
|
|||||||
@@ -4,6 +4,7 @@
|
|||||||
types / modalities.
|
types / modalities.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import itertools
|
||||||
from pathlib import PosixPath
|
from pathlib import PosixPath
|
||||||
|
|
||||||
from .....conftest import (
|
from .....conftest import (
|
||||||
@@ -27,9 +28,14 @@ def run_single_image_test(
|
|||||||
vllm_runner: type[VllmRunner],
|
vllm_runner: type[VllmRunner],
|
||||||
image_assets: ImageTestAssets,
|
image_assets: ImageTestAssets,
|
||||||
):
|
):
|
||||||
assert test_case.size_wrapper is not None
|
assert test_case.size_wrappers
|
||||||
inputs = builders.build_single_image_inputs_from_test_info(
|
inputs = list(
|
||||||
model_test_info, image_assets, test_case.size_wrapper, tmp_path
|
itertools.chain.from_iterable(
|
||||||
|
builders.build_single_image_inputs_from_test_info(
|
||||||
|
model_test_info, image_assets, size_wrapper, tmp_path
|
||||||
|
)
|
||||||
|
for size_wrapper in test_case.size_wrappers
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
core.run_test(
|
core.run_test(
|
||||||
@@ -55,9 +61,14 @@ def run_multi_image_test(
|
|||||||
vllm_runner: type[VllmRunner],
|
vllm_runner: type[VllmRunner],
|
||||||
image_assets: ImageTestAssets,
|
image_assets: ImageTestAssets,
|
||||||
):
|
):
|
||||||
assert test_case.size_wrapper is not None
|
assert test_case.size_wrappers
|
||||||
inputs = builders.build_multi_image_inputs_from_test_info(
|
inputs = list(
|
||||||
model_test_info, image_assets, test_case.size_wrapper, tmp_path
|
itertools.chain.from_iterable(
|
||||||
|
builders.build_multi_image_inputs_from_test_info(
|
||||||
|
model_test_info, image_assets, size_wrapper, tmp_path
|
||||||
|
)
|
||||||
|
for size_wrapper in test_case.size_wrappers
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
core.run_test(
|
core.run_test(
|
||||||
@@ -82,9 +93,20 @@ def run_embedding_test(
|
|||||||
vllm_runner: type[VllmRunner],
|
vllm_runner: type[VllmRunner],
|
||||||
image_assets: ImageTestAssets,
|
image_assets: ImageTestAssets,
|
||||||
):
|
):
|
||||||
assert test_case.size_wrapper is not None
|
assert test_case.size_wrappers
|
||||||
inputs, vllm_embeddings = builders.build_embedding_inputs_from_test_info(
|
inputs_and_embeddings = [
|
||||||
model_test_info, image_assets, test_case.size_wrapper
|
builders.build_embedding_inputs_from_test_info(
|
||||||
|
model_test_info, image_assets, size_wrapper
|
||||||
|
)
|
||||||
|
for size_wrapper in test_case.size_wrappers
|
||||||
|
]
|
||||||
|
inputs = list(
|
||||||
|
itertools.chain.from_iterable(inputs for inputs, _ in inputs_and_embeddings)
|
||||||
|
)
|
||||||
|
vllm_embeddings = list(
|
||||||
|
itertools.chain.from_iterable(
|
||||||
|
embeddings for _, embeddings in inputs_and_embeddings
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
core.run_test(
|
core.run_test(
|
||||||
@@ -110,14 +132,19 @@ def run_video_test(
|
|||||||
vllm_runner: type[VllmRunner],
|
vllm_runner: type[VllmRunner],
|
||||||
video_assets: VideoTestAssets,
|
video_assets: VideoTestAssets,
|
||||||
):
|
):
|
||||||
assert test_case.size_wrapper is not None
|
assert test_case.size_wrappers
|
||||||
assert test_case.num_video_frames is not None
|
assert test_case.num_video_frames is not None
|
||||||
inputs = builders.build_video_inputs_from_test_info(
|
inputs = list(
|
||||||
model_test_info,
|
itertools.chain.from_iterable(
|
||||||
video_assets,
|
builders.build_video_inputs_from_test_info(
|
||||||
test_case.size_wrapper,
|
model_test_info,
|
||||||
test_case.num_video_frames,
|
video_assets,
|
||||||
test_case.needs_video_metadata,
|
size_wrapper,
|
||||||
|
test_case.num_video_frames,
|
||||||
|
test_case.needs_video_metadata,
|
||||||
|
)
|
||||||
|
for size_wrapper in test_case.size_wrappers
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
core.run_test(
|
core.run_test(
|
||||||
|
|||||||
@@ -158,11 +158,9 @@ class VLMTestInfo(NamedTuple):
|
|||||||
num_video_frames: int | tuple[int] = 16
|
num_video_frames: int | tuple[int] = 16
|
||||||
needs_video_metadata: bool = False
|
needs_video_metadata: bool = False
|
||||||
|
|
||||||
# Fixed image sizes / image size factors; most tests use image_size_factors
|
# Fixed image sizes / image size factors; most tests use image_size_factors.
|
||||||
# The values provided for these two fields will be stacked and expanded
|
# Each inner iterable defines one request batch. All batches are run against
|
||||||
# such that each model will consider each image size factor / image size
|
# the same model instance.
|
||||||
# once per tests (much like concatenating and wrapping in one parametrize
|
|
||||||
# call)
|
|
||||||
image_size_factors: Iterable[Iterable[float]] = IMAGE_SIZE_FACTORS
|
image_size_factors: Iterable[Iterable[float]] = IMAGE_SIZE_FACTORS
|
||||||
image_sizes: Iterable[Iterable[tuple[int, int]]] | None = None
|
image_sizes: Iterable[Iterable[tuple[int, int]]] | None = None
|
||||||
|
|
||||||
@@ -211,8 +209,8 @@ class ExpandableVLMTestArgs(NamedTuple):
|
|||||||
num_logprobs: int
|
num_logprobs: int
|
||||||
dtype: str
|
dtype: str
|
||||||
distributed_executor_backend: str | None
|
distributed_executor_backend: str | None
|
||||||
# Sizes are used for everything except for custom input tests
|
# Sizes are used for everything except audio and custom input tests.
|
||||||
size_wrapper: ImageSizeWrapper | None = None
|
size_wrappers: tuple[ImageSizeWrapper, ...] = ()
|
||||||
# Video only
|
# Video only
|
||||||
num_video_frames: int | None = None
|
num_video_frames: int | None = None
|
||||||
needs_video_metadata: bool = False
|
needs_video_metadata: bool = False
|
||||||
|
|||||||
Reference in New Issue
Block a user