From f1816fb1920c1746c483bd1b67238d1cc85de46f Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Fri, 13 Mar 2026 14:16:02 -0700 Subject: [PATCH] [CI] Split V1 e2e + engine (1 GPU) into separate jobs (#36945) Co-authored-by: Claude Opus 4.6 --- .buildkite/test-amd.yaml | 12 ++--- .buildkite/test_areas/engine.yaml | 44 ++++++++++--------- .buildkite/test_areas/model_runner_v2.yaml | 10 ++--- .buildkite/test_areas/spec_decode.yaml | 40 +++++++++++++++++ tests/v1/e2e/general/__init__.py | 0 .../{ => general}/test_async_scheduling.py | 4 +- .../{ => general}/test_cascade_attention.py | 2 +- .../e2e/{ => general}/test_context_length.py | 0 .../test_correctness_sliding_window.py | 2 +- .../test_kv_sharing_fast_prefill.py | 4 +- .../{ => general}/test_mamba_prefix_cache.py | 0 tests/v1/e2e/{ => general}/test_min_tokens.py | 2 +- .../test_pooling_chunked_prefill.py | 0 .../e2e/{ => general}/test_streaming_input.py | 0 tests/v1/e2e/spec_decode/__init__.py | 0 .../test_async_spec_decode.py | 0 .../test_lora_with_spec_decode.py | 0 .../e2e/{ => spec_decode}/test_spec_decode.py | 0 18 files changed, 81 insertions(+), 39 deletions(-) create mode 100644 .buildkite/test_areas/spec_decode.yaml create mode 100644 tests/v1/e2e/general/__init__.py rename tests/v1/e2e/{ => general}/test_async_scheduling.py (99%) rename tests/v1/e2e/{ => general}/test_cascade_attention.py (95%) rename tests/v1/e2e/{ => general}/test_context_length.py (100%) rename tests/v1/e2e/{ => general}/test_correctness_sliding_window.py (98%) rename tests/v1/e2e/{ => general}/test_kv_sharing_fast_prefill.py (95%) rename tests/v1/e2e/{ => general}/test_mamba_prefix_cache.py (100%) rename tests/v1/e2e/{ => general}/test_min_tokens.py (99%) rename tests/v1/e2e/{ => general}/test_pooling_chunked_prefill.py (100%) rename tests/v1/e2e/{ => general}/test_streaming_input.py (100%) create mode 100644 tests/v1/e2e/spec_decode/__init__.py rename tests/v1/e2e/{ => spec_decode}/test_async_spec_decode.py (100%) rename tests/v1/e2e/{ => spec_decode}/test_lora_with_spec_decode.py (100%) rename tests/v1/e2e/{ => spec_decode}/test_spec_decode.py (100%) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 829743d5cf..7f8020540a 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -369,7 +369,7 @@ steps: - vllm/ - tests/v1 commands: - - pytest -v -s v1/e2e/test_spec_decode.py -k "tensor_parallelism" + - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "tensor_parallelism" - label: V1 Test e2e (4 GPUs) # 65min timeout_in_minutes: 90 @@ -380,7 +380,7 @@ steps: - vllm/ - tests/v1 commands: - - pytest -v -s v1/e2e/test_spec_decode.py -k "eagle_correctness_heavy" + - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "eagle_correctness_heavy" - label: V1 Test entrypoints # 35min timeout_in_minutes: 50 @@ -1744,7 +1744,7 @@ steps: - tests/v1 commands: # Only run tests that need exactly 2 GPUs - - pytest -v -s v1/e2e/test_spec_decode.py -k "tensor_parallelism" + - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "tensor_parallelism" - label: V1 Test e2e (4 GPUs) # 65min timeout_in_minutes: 90 @@ -1759,7 +1759,7 @@ steps: - tests/v1 commands: # Only run tests that need 4 GPUs - - pytest -v -s v1/e2e/test_spec_decode.py -k "eagle_correctness_heavy" + - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "eagle_correctness_heavy" - label: V1 Test entrypoints # 35min timeout_in_minutes: 50 @@ -3494,7 +3494,7 @@ steps: - tests/v1 commands: # Only run tests that need exactly 2 GPUs - - pytest -v -s v1/e2e/test_spec_decode.py -k "tensor_parallelism" + - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "tensor_parallelism" - label: V1 Test e2e (4 GPUs) # 65min timeout_in_minutes: 90 @@ -3509,7 +3509,7 @@ steps: - tests/v1 commands: # Only run tests that need 4 GPUs - - pytest -v -s v1/e2e/test_spec_decode.py -k "eagle_correctness_heavy" + - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "eagle_correctness_heavy" - label: V1 Test entrypoints # 35min timeout_in_minutes: 50 diff --git a/.buildkite/test_areas/engine.yaml b/.buildkite/test_areas/engine.yaml index b5b3eeb6d7..be83bab8fa 100644 --- a/.buildkite/test_areas/engine.yaml +++ b/.buildkite/test_areas/engine.yaml @@ -1,5 +1,5 @@ group: Engine -depends_on: +depends_on: - image-build steps: - label: Engine @@ -14,28 +14,30 @@ steps: commands: - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py -- label: V1 e2e + engine (1 GPU) - timeout_in_minutes: 45 +- label: Engine (1 GPU) + timeout_in_minutes: 30 source_file_dependencies: - - vllm/ - - tests/v1 + - vllm/v1/engine/ + - tests/v1/engine/ commands: - # TODO: accuracy does not match, whether setting - # VLLM_USE_FLASHINFER_SAMPLER or not on H100. - - pytest -v -s v1/e2e - # Run this test standalone for now; - # need to untangle use (implicit) use of spawn/fork across the tests. - pytest -v -s v1/engine/test_preprocess_error_handling.py - # Run the rest of v1/engine tests - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py - mirror: - amd: - device: mi325_1 - depends_on: - - image-build-amd - commands: - - pytest -v -s v1/e2e - - pytest -v -s v1/engine + +- label: e2e Scheduling (1 GPU) + timeout_in_minutes: 30 + source_file_dependencies: + - vllm/v1/ + - tests/v1/e2e/general/ + commands: + - pytest -v -s v1/e2e/general/test_async_scheduling.py + +- label: e2e Core (1 GPU) + timeout_in_minutes: 30 + source_file_dependencies: + - vllm/v1/ + - tests/v1/e2e/general/ + commands: + - pytest -v -s v1/e2e/general --ignore v1/e2e/general/test_async_scheduling.py - label: V1 e2e (2 GPUs) timeout_in_minutes: 60 # TODO: Fix timeout after we have more confidence in the test stability @@ -46,7 +48,7 @@ steps: - tests/v1/e2e commands: # Only run tests that need exactly 2 GPUs - - pytest -v -s v1/e2e/test_spec_decode.py -k "tensor_parallelism" + - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "tensor_parallelism" mirror: amd: device: mi325_2 @@ -62,7 +64,7 @@ steps: - tests/v1/e2e commands: # Only run tests that need 4 GPUs - - pytest -v -s v1/e2e/test_spec_decode.py -k "eagle_correctness_heavy" + - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "eagle_correctness_heavy" mirror: amd: device: mi325_4 diff --git a/.buildkite/test_areas/model_runner_v2.yaml b/.buildkite/test_areas/model_runner_v2.yaml index e19b7297f0..85421399d1 100644 --- a/.buildkite/test_areas/model_runner_v2.yaml +++ b/.buildkite/test_areas/model_runner_v2.yaml @@ -18,9 +18,9 @@ steps: - pytest -v -s v1/engine/test_llm_engine.py -k "not test_engine_metrics" # This requires eager until we sort out CG correctness issues. # TODO: remove ENFORCE_EAGER here after https://github.com/vllm-project/vllm/pull/32936 is merged. - - ENFORCE_EAGER=1 pytest -v -s v1/e2e/test_async_scheduling.py -k "not ngram" - - pytest -v -s v1/e2e/test_context_length.py - - pytest -v -s v1/e2e/test_min_tokens.py + - ENFORCE_EAGER=1 pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram" + - pytest -v -s v1/e2e/general/test_context_length.py + - pytest -v -s v1/e2e/general/test_min_tokens.py # Temporary hack filter to exclude ngram spec decoding based tests. - pytest -v -s v1/entrypoints/llm/test_struct_output_generate.py -k "xgrammar and not speculative_config6 and not speculative_config7 and not speculative_config8 and not speculative_config0" @@ -102,9 +102,9 @@ steps: - vllm/v1/worker/gpu/ - vllm/v1/worker/gpu_worker.py - tests/v1/spec_decode/test_max_len.py - - tests/v1/e2e/test_spec_decode.py + - tests/v1/e2e/spec_decode/test_spec_decode.py commands: - set -x - export VLLM_USE_V2_MODEL_RUNNER=1 - pytest -v -s v1/spec_decode/test_max_len.py -k "eagle or mtp" - - pytest -v -s v1/e2e/test_spec_decode.py -k "eagle or mtp" + - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "eagle or mtp" diff --git a/.buildkite/test_areas/spec_decode.yaml b/.buildkite/test_areas/spec_decode.yaml new file mode 100644 index 0000000000..8dba7a2f8c --- /dev/null +++ b/.buildkite/test_areas/spec_decode.yaml @@ -0,0 +1,40 @@ +group: Spec Decode +depends_on: + - image-build +steps: +- label: Spec Decode Eagle + timeout_in_minutes: 30 + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - tests/v1/e2e/spec_decode/ + commands: + - pytest -v -s v1/e2e/spec_decode -k "eagle_correctness" + +- label: Spec Decode Speculators + MTP + timeout_in_minutes: 30 + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/transformers_utils/configs/speculators/ + - tests/v1/e2e/spec_decode/ + commands: + - pytest -v -s v1/e2e/spec_decode -k "speculators or mtp_correctness" + +- label: Spec Decode Ngram + Suffix + timeout_in_minutes: 30 + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - tests/v1/e2e/spec_decode/ + commands: + - pytest -v -s v1/e2e/spec_decode -k "ngram or suffix" + +- label: Spec Decode Draft Model + timeout_in_minutes: 30 + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - tests/v1/e2e/spec_decode/ + commands: + - pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference" diff --git a/tests/v1/e2e/general/__init__.py b/tests/v1/e2e/general/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/v1/e2e/test_async_scheduling.py b/tests/v1/e2e/general/test_async_scheduling.py similarity index 99% rename from tests/v1/e2e/test_async_scheduling.py rename to tests/v1/e2e/general/test_async_scheduling.py index a54b612f77..acb08997c7 100644 --- a/tests/v1/e2e/test_async_scheduling.py +++ b/tests/v1/e2e/general/test_async_scheduling.py @@ -14,8 +14,8 @@ from vllm.platforms import current_platform from vllm.sampling_params import StructuredOutputsParams from vllm.v1.metrics.reader import Metric -from ...conftest import VllmRunner -from ...models.utils import check_outputs_equal +from ....conftest import VllmRunner +from ....models.utils import check_outputs_equal MODEL = "Qwen/Qwen3-0.6B" MTP_MODEL = "meta-llama/Llama-3.2-1B-Instruct" diff --git a/tests/v1/e2e/test_cascade_attention.py b/tests/v1/e2e/general/test_cascade_attention.py similarity index 95% rename from tests/v1/e2e/test_cascade_attention.py rename to tests/v1/e2e/general/test_cascade_attention.py index a7be981805..be889b3869 100644 --- a/tests/v1/e2e/test_cascade_attention.py +++ b/tests/v1/e2e/general/test_cascade_attention.py @@ -5,7 +5,7 @@ import pytest from vllm import LLM, SamplingParams -from ...utils import create_new_process_for_each_test +from ....utils import create_new_process_for_each_test @create_new_process_for_each_test() diff --git a/tests/v1/e2e/test_context_length.py b/tests/v1/e2e/general/test_context_length.py similarity index 100% rename from tests/v1/e2e/test_context_length.py rename to tests/v1/e2e/general/test_context_length.py diff --git a/tests/v1/e2e/test_correctness_sliding_window.py b/tests/v1/e2e/general/test_correctness_sliding_window.py similarity index 98% rename from tests/v1/e2e/test_correctness_sliding_window.py rename to tests/v1/e2e/general/test_correctness_sliding_window.py index b6a78eaa09..01d6044417 100644 --- a/tests/v1/e2e/test_correctness_sliding_window.py +++ b/tests/v1/e2e/general/test_correctness_sliding_window.py @@ -7,7 +7,7 @@ import pytest from vllm import LLM, SamplingParams from vllm.platforms import current_platform -from ...utils import check_answers, prep_prompts +from ....utils import check_answers, prep_prompts @dataclass diff --git a/tests/v1/e2e/test_kv_sharing_fast_prefill.py b/tests/v1/e2e/general/test_kv_sharing_fast_prefill.py similarity index 95% rename from tests/v1/e2e/test_kv_sharing_fast_prefill.py rename to tests/v1/e2e/general/test_kv_sharing_fast_prefill.py index 92b4d4532e..4bb8d63a8a 100644 --- a/tests/v1/e2e/test_kv_sharing_fast_prefill.py +++ b/tests/v1/e2e/general/test_kv_sharing_fast_prefill.py @@ -9,7 +9,7 @@ from vllm import LLM, SamplingParams from vllm.config import CompilationConfig, CompilationMode from vllm.platforms import current_platform -from ...utils import check_answers, fork_new_process_for_each_test, prep_prompts +from ....utils import check_answers, fork_new_process_for_each_test, prep_prompts # global seed SEED = 42 @@ -18,7 +18,7 @@ SEED = 42 @pytest.fixture def test_prompts(): """ - Adapted from tests/v1/e2e/test_spec_decode.py + Adapted from tests/v1/e2e/spec_decode/test_spec_decode.py """ prompt_types = ["repeat", "sentence"] # Setting higher num prompts increases the chance of numerics mismatch diff --git a/tests/v1/e2e/test_mamba_prefix_cache.py b/tests/v1/e2e/general/test_mamba_prefix_cache.py similarity index 100% rename from tests/v1/e2e/test_mamba_prefix_cache.py rename to tests/v1/e2e/general/test_mamba_prefix_cache.py diff --git a/tests/v1/e2e/test_min_tokens.py b/tests/v1/e2e/general/test_min_tokens.py similarity index 99% rename from tests/v1/e2e/test_min_tokens.py rename to tests/v1/e2e/general/test_min_tokens.py index ec7ee0c3eb..bb041cd386 100644 --- a/tests/v1/e2e/test_min_tokens.py +++ b/tests/v1/e2e/general/test_min_tokens.py @@ -497,6 +497,6 @@ if __name__ == "__main__": Usage: cd vllm/ - python -m pytest tests/v1/e2e/test_min_tokens.py -v + python -m pytest tests/v1/e2e/general/test_min_tokens.py -v """ pytest.main([__file__, "-v"]) diff --git a/tests/v1/e2e/test_pooling_chunked_prefill.py b/tests/v1/e2e/general/test_pooling_chunked_prefill.py similarity index 100% rename from tests/v1/e2e/test_pooling_chunked_prefill.py rename to tests/v1/e2e/general/test_pooling_chunked_prefill.py diff --git a/tests/v1/e2e/test_streaming_input.py b/tests/v1/e2e/general/test_streaming_input.py similarity index 100% rename from tests/v1/e2e/test_streaming_input.py rename to tests/v1/e2e/general/test_streaming_input.py diff --git a/tests/v1/e2e/spec_decode/__init__.py b/tests/v1/e2e/spec_decode/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/v1/e2e/test_async_spec_decode.py b/tests/v1/e2e/spec_decode/test_async_spec_decode.py similarity index 100% rename from tests/v1/e2e/test_async_spec_decode.py rename to tests/v1/e2e/spec_decode/test_async_spec_decode.py diff --git a/tests/v1/e2e/test_lora_with_spec_decode.py b/tests/v1/e2e/spec_decode/test_lora_with_spec_decode.py similarity index 100% rename from tests/v1/e2e/test_lora_with_spec_decode.py rename to tests/v1/e2e/spec_decode/test_lora_with_spec_decode.py diff --git a/tests/v1/e2e/test_spec_decode.py b/tests/v1/e2e/spec_decode/test_spec_decode.py similarity index 100% rename from tests/v1/e2e/test_spec_decode.py rename to tests/v1/e2e/spec_decode/test_spec_decode.py