diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index 7f97c42d1d..bc775dbf4e 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -435,6 +435,7 @@ steps: - pip install pytest-timeout pytest-forked - pytest -v -s v1/determinism/test_batch_invariance.py - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py + - pytest -v -s v1/determinism/test_batch_invariance_vlm.py - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA - VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k FLASH_ATTN - pytest -v -s v1/determinism/test_nvfp4_batch_invariant.py diff --git a/docs/features/batch_invariance.md b/docs/features/batch_invariance.md index 3171d26ce8..ddfef8efbd 100644 --- a/docs/features/batch_invariance.md +++ b/docs/features/batch_invariance.md @@ -102,6 +102,7 @@ Batch invariance has been tested and verified on the following models: - **DeepSeek series**: `deepseek-ai/DeepSeek-V3`, `deepseek-ai/DeepSeek-V3-0324`, `deepseek-ai/DeepSeek-R1`, `deepseek-ai/DeepSeek-V3.1` - **Qwen3 (Dense)**: `Qwen/Qwen3-1.7B`, `Qwen/Qwen3-8B`, `Qwen/Qwen3-4B-AWQ`, `Qwen/Qwen3-8B-AWQ` +- **Qwen3-VL (Vision-Language)**: `Qwen/Qwen3-VL-2B-Instruct`, `Qwen/Qwen3-VL-4B-Instruct` (single image and video inputs) - **Qwen3 (MoE)**: `Qwen/Qwen3-30B-A3B`, `Qwen/Qwen3-Next-80B-A3B-Instruct`, `Qwen/Qwen3-30B-A3B-Thinking-2507-FP8` - **Qwen2.5**: `Qwen/Qwen2.5-0.5B-Instruct`, `Qwen/Qwen2.5-1.5B-Instruct`, `Qwen/Qwen2.5-3B-Instruct`, `Qwen/Qwen2.5-7B-Instruct`, `Qwen/Qwen2.5-14B-Instruct`, `Qwen/Qwen2.5-32B-Instruct` - **Llama 3**: Llama3.1 and 3.2 series, `meta-llama/Llama-3.2-3B-Instruct` for example diff --git a/tests/v1/determinism/test_batch_invariance_vlm.py b/tests/v1/determinism/test_batch_invariance_vlm.py new file mode 100644 index 0000000000..266768f672 --- /dev/null +++ b/tests/v1/determinism/test_batch_invariance_vlm.py @@ -0,0 +1,103 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Batch-invariance regression tests for vision-language models (VLMs).""" + +import contextlib +import os + +import numpy as np +import pytest +import torch +from PIL import Image +from utils import _extract_step_logprobs, skip_if_not_cuda + +from vllm import LLM, SamplingParams + +VLM_TEST_MODEL = os.getenv("VLLM_VLM_TEST_MODEL", "Qwen/Qwen3-VL-2B-Instruct") + +IMAGE_PROMPT = ( + "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>" + "Describe this image.<|im_end|><|im_start|>assistant\n" +) +VIDEO_PROMPT = ( + "<|im_start|>user\n<|vision_start|><|video_pad|><|vision_end|>" + "Describe this video.<|im_end|><|im_start|>assistant\n" +) + + +def _make_image(seed: int, size: int) -> Image.Image: + rng = np.random.default_rng(seed) + return Image.fromarray(rng.integers(0, 255, (size, size, 3), dtype=np.uint8)) + + +def _make_inputs(input_type: str, num_reqs: int) -> list[dict]: + inputs = [] + for i in range(num_reqs): + if input_type == "image": + prompt = IMAGE_PROMPT + mm_data = {"image": _make_image(1000 + i, size=256)} + else: + frames = [_make_image(1000 + i + f, size=128) for f in range(4)] + prompt = VIDEO_PROMPT + mm_data = { + "video": ( + frames, + { + "total_num_frames": len(frames), + "fps": 1.0, + "frames_indices": list(range(len(frames))), + "do_sample_frames": False, + }, + ) + } + inputs.append({"prompt": prompt, "multi_modal_data": mm_data}) + return inputs + + +def _assert_batch_invariant(llm: LLM, inputs: list[dict], sampling) -> None: + bs1 = [ + _extract_step_logprobs(llm.generate([inp], sampling, use_tqdm=False)[0]) + for inp in inputs + ] + bsN = [_extract_step_logprobs(out) for out in llm.generate(inputs, sampling)] + + failures = [] + for i, ((lp1, t1), (lpN, tN)) in enumerate(zip(bs1, bsN)): + if lp1 is None or lpN is None: + pytest.skip( + "Logits are not available on RequestOutput; " + "enable logprobs return to run this test." + ) + if t1 != tN: + failures.append(f"req {i}: token mismatch bs1={t1} bsN={tN}") + elif not torch.equal(lp1, lpN): + d = (lp1 - lpN).abs().max().item() + failures.append(f"req {i}: logprob mismatch max_diff={d:.3e}") + + if failures: + pytest.fail( + f"Batch invariance violated for {len(failures)}/{len(inputs)} " + f"requests:\n" + "\n".join(failures) + ) + + +@skip_if_not_cuda +@pytest.mark.parametrize("input_type", ["image", "video"]) +@pytest.mark.parametrize("mm_encoder_attn_backend", ["FLASH_ATTN", "TORCH_SDPA"]) +def test_vlm_batch_invariance_bs1_vs_bsN(input_type: str, mm_encoder_attn_backend: str): + inputs = _make_inputs(input_type, num_reqs=4) + sampling = SamplingParams(temperature=0.0, max_tokens=8, seed=1234, logprobs=5) + + llm = LLM( + model=VLM_TEST_MODEL, + dtype="bfloat16", + enforce_eager=True, + max_num_seqs=16, + gpu_memory_utilization=0.85, + mm_encoder_attn_backend=mm_encoder_attn_backend, + ) + try: + _assert_batch_invariant(llm, inputs, sampling) + finally: + with contextlib.suppress(Exception): + llm.shutdown()