From 0ab67c02225c18ed664864f75c3a38c0659f4be1 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Wed, 29 Apr 2026 16:59:16 -0700 Subject: [PATCH] [CI] Add key field to all test_areas pipeline steps (#41201) Signed-off-by: khluu Co-authored-by: Claude --- .buildkite/test_areas/attention.yaml | 2 ++ .buildkite/test_areas/basic_correctness.yaml | 1 + .buildkite/test_areas/benchmarks.yaml | 2 ++ .buildkite/test_areas/compile.yaml | 13 +++++++++++++ .buildkite/test_areas/cuda.yaml | 2 ++ .buildkite/test_areas/disaggregated.yaml | 8 ++++++++ .buildkite/test_areas/distributed.yaml | 14 ++++++++++++++ .buildkite/test_areas/e2e_integration.yaml | 5 +++++ .buildkite/test_areas/engine.yaml | 7 +++++++ .buildkite/test_areas/entrypoints.yaml | 11 ++++++++++- .buildkite/test_areas/expert_parallelism.yaml | 3 +++ .buildkite/test_areas/kernels.yaml | 15 +++++++++++++++ .buildkite/test_areas/lm_eval.yaml | 11 +++++++++++ .buildkite/test_areas/lora.yaml | 2 ++ .buildkite/test_areas/misc.yaml | 15 ++++++++++++++- .buildkite/test_areas/model_executor.yaml | 1 + .buildkite/test_areas/model_runner_v2.yaml | 5 +++++ .buildkite/test_areas/models_basic.yaml | 6 ++++++ .buildkite/test_areas/models_distributed.yaml | 1 + .buildkite/test_areas/models_language.yaml | 7 +++++++ .buildkite/test_areas/models_multimodal.yaml | 13 ++++++++++++- .buildkite/test_areas/plugins.yaml | 1 + .buildkite/test_areas/pytorch.yaml | 6 ++++++ .buildkite/test_areas/quantization.yaml | 3 +++ .buildkite/test_areas/ray_compat.yaml | 1 + .buildkite/test_areas/samplers.yaml | 1 + .buildkite/test_areas/spec_decode.yaml | 8 ++++++++ .buildkite/test_areas/weight_loading.yaml | 1 + 28 files changed, 162 insertions(+), 3 deletions(-) diff --git a/.buildkite/test_areas/attention.yaml b/.buildkite/test_areas/attention.yaml index 4bcf116f27..9b1f67705d 100644 --- a/.buildkite/test_areas/attention.yaml +++ b/.buildkite/test_areas/attention.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: V1 attention (H100) + key: v1-attention-h100 timeout_in_minutes: 30 device: h100 source_file_dependencies: @@ -14,6 +15,7 @@ steps: - pytest -v -s v1/attention - label: V1 attention (B200) + key: v1-attention-b200 timeout_in_minutes: 30 device: b200 source_file_dependencies: diff --git a/.buildkite/test_areas/basic_correctness.yaml b/.buildkite/test_areas/basic_correctness.yaml index 042734e843..5d547cd486 100644 --- a/.buildkite/test_areas/basic_correctness.yaml +++ b/.buildkite/test_areas/basic_correctness.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: Basic Correctness + key: basic-correctness timeout_in_minutes: 30 device: h200_18gb source_file_dependencies: diff --git a/.buildkite/test_areas/benchmarks.yaml b/.buildkite/test_areas/benchmarks.yaml index 4cda6fff14..3385f9472a 100644 --- a/.buildkite/test_areas/benchmarks.yaml +++ b/.buildkite/test_areas/benchmarks.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: Benchmarks CLI Test + key: benchmarks-cli-test timeout_in_minutes: 20 device: h200_18gb source_file_dependencies: @@ -12,6 +13,7 @@ steps: - pytest -v -s benchmarks/ - label: Attention Benchmarks Smoke Test (B200) + key: attention-benchmarks-smoke-test-b200 device: b200 num_gpus: 2 optional: true diff --git a/.buildkite/test_areas/compile.yaml b/.buildkite/test_areas/compile.yaml index aa46447c24..638e5b0eec 100644 --- a/.buildkite/test_areas/compile.yaml +++ b/.buildkite/test_areas/compile.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: Sequence Parallel Correctness Tests (2 GPUs) + key: sequence-parallel-correctness-tests-2-gpus timeout_in_minutes: 50 working_dir: "/vllm-workspace/" num_devices: 2 @@ -17,6 +18,7 @@ steps: - pytest -v -s tests/compile/correctness_e2e/test_sequence_parallel.py - label: Sequence Parallel Correctness Tests (2xH100) + key: sequence-parallel-correctness-tests-2xh100 timeout_in_minutes: 50 working_dir: "/vllm-workspace/" device: h100 @@ -27,6 +29,7 @@ steps: - pytest -v -s tests/compile/correctness_e2e/test_sequence_parallel.py - label: AsyncTP Correctness Tests (2xH100) + key: asynctp-correctness-tests-2xh100 timeout_in_minutes: 50 working_dir: "/vllm-workspace/" device: h100 @@ -37,6 +40,7 @@ steps: - pytest -v -s tests/compile/correctness_e2e/test_async_tp.py - label: AsyncTP Correctness Tests (B200) + key: asynctp-correctness-tests-b200 timeout_in_minutes: 50 working_dir: "/vllm-workspace/" device: b200 @@ -47,6 +51,7 @@ steps: - pytest -v -s tests/compile/correctness_e2e/test_async_tp.py - label: Distributed Compile Unit Tests (2xH100) + key: distributed-compile-unit-tests-2xh100 timeout_in_minutes: 20 working_dir: "/vllm-workspace/" device: h100 @@ -60,6 +65,7 @@ steps: - pytest -s -v tests/compile/passes/distributed - label: Fusion and Compile Unit Tests (2xB200) + key: fusion-and-compile-unit-tests-2xb200 timeout_in_minutes: 20 working_dir: "/vllm-workspace/" device: b200 @@ -89,6 +95,7 @@ steps: - pytest -v -s tests/compile/fullgraph/test_full_graph.py::test_fp8_kv_scale_compile - label: Fusion E2E Quick (H100) + key: fusion-e2e-quick-h100 timeout_in_minutes: 15 working_dir: "/vllm-workspace/" device: h100 @@ -107,6 +114,7 @@ steps: - pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "inductor_partition and not +rms_norm and +quant_fp8 and (qwen3 or deepseek)" - label: Fusion E2E Config Sweep (H100) + key: fusion-e2e-config-sweep-h100 timeout_in_minutes: 30 working_dir: "/vllm-workspace/" device: h100 @@ -126,6 +134,7 @@ steps: - pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "llama-3" - label: Fusion E2E Config Sweep (B200) + key: fusion-e2e-config-sweep-b200 timeout_in_minutes: 30 working_dir: "/vllm-workspace/" device: b200 @@ -139,6 +148,7 @@ steps: - pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "inductor_partition and (FLASHINFER and not +rms_norm and (not +quant_fp8 or +quant_fp8 and (qwen3 or deepseek)) or llama-3)" - label: Fusion E2E TP2 Quick (H100) + key: fusion-e2e-tp2-quick-h100 timeout_in_minutes: 20 working_dir: "/vllm-workspace/" device: h100 @@ -156,6 +166,7 @@ steps: - pytest -v -s tests/compile/fusions_e2e/test_tp2_async_tp.py -k "inductor_partition and not +rms_norm and (not +quant_fp8 or +quant_fp8 and (qwen3 or deepseek))" - label: Fusion E2E TP2 AR-RMS Config Sweep (H100) + key: fusion-e2e-tp2-ar-rms-config-sweep-h100 timeout_in_minutes: 40 working_dir: "/vllm-workspace/" device: h100 @@ -175,6 +186,7 @@ steps: - pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py -k "llama-3" - label: Fusion E2E TP2 AsyncTP Config Sweep (H100) + key: fusion-e2e-tp2-asynctp-config-sweep-h100 timeout_in_minutes: 40 working_dir: "/vllm-workspace/" device: h100 @@ -194,6 +206,7 @@ steps: - pytest -v -s tests/compile/fusions_e2e/test_tp2_async_tp.py -k "llama-3" - label: Fusion E2E TP2 (B200) + key: fusion-e2e-tp2-b200 timeout_in_minutes: 20 working_dir: "/vllm-workspace/" device: b200 diff --git a/.buildkite/test_areas/cuda.yaml b/.buildkite/test_areas/cuda.yaml index 4d1efdb13c..9cb19ac408 100644 --- a/.buildkite/test_areas/cuda.yaml +++ b/.buildkite/test_areas/cuda.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: Platform Tests (CUDA) + key: platform-tests-cuda timeout_in_minutes: 15 device: h200_18gb source_file_dependencies: @@ -13,6 +14,7 @@ steps: - pytest -v -s cuda/test_platform_no_cuda_init.py - label: Cudagraph + key: cudagraph timeout_in_minutes: 20 source_file_dependencies: - tests/v1/cudagraph diff --git a/.buildkite/test_areas/disaggregated.yaml b/.buildkite/test_areas/disaggregated.yaml index a10fda41ef..cef2c69d77 100644 --- a/.buildkite/test_areas/disaggregated.yaml +++ b/.buildkite/test_areas/disaggregated.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: Distributed NixlConnector PD accuracy (4 GPUs) + key: distributed-nixlconnector-pd-accuracy-4-gpus timeout_in_minutes: 30 working_dir: "/vllm-workspace/tests" num_devices: 4 @@ -13,6 +14,7 @@ steps: - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt - bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh - label: Distributed FlashInfer NixlConnector PD accuracy (4 GPUs) + key: distributed-flashinfer-nixlconnector-pd-accuracy-4-gpus timeout_in_minutes: 30 working_dir: "/vllm-workspace/tests" num_devices: 4 @@ -24,6 +26,7 @@ steps: - FLASHINFER=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh - label: DP EP Distributed NixlConnector PD accuracy tests (4 GPUs) + key: dp-ep-distributed-nixlconnector-pd-accuracy-tests-4-gpus timeout_in_minutes: 30 working_dir: "/vllm-workspace/tests" num_devices: 4 @@ -35,6 +38,7 @@ steps: - DP_EP=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh - label: CrossLayer KV layout Distributed NixlConnector PD accuracy tests (4 GPUs) + key: crosslayer-kv-layout-distributed-nixlconnector-pd-accuracy-tests-4-gpus timeout_in_minutes: 30 working_dir: "/vllm-workspace/tests" num_devices: 4 @@ -46,6 +50,7 @@ steps: - CROSS_LAYERS_BLOCKS=True bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh - label: Hybrid SSM NixlConnector PD accuracy tests (4 GPUs) + key: hybrid-ssm-nixlconnector-pd-accuracy-tests-4-gpus timeout_in_minutes: 20 working_dir: "/vllm-workspace/tests" num_devices: 4 @@ -57,6 +62,7 @@ steps: - HYBRID_SSM=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh - label: MultiConnector (Nixl+Offloading) PD accuracy (2 GPUs) + key: multiconnector-nixl-offloading-pd-accuracy-2-gpus timeout_in_minutes: 30 working_dir: "/vllm-workspace/tests" num_devices: 2 @@ -71,6 +77,7 @@ steps: - bash v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh - label: NixlConnector PD + Spec Decode acceptance (2 GPUs) + key: nixlconnector-pd-spec-decode-acceptance-2-gpus timeout_in_minutes: 30 device: a100 working_dir: "/vllm-workspace/tests" @@ -84,6 +91,7 @@ steps: - bash v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh - label: MultiConnector (Nixl+Offloading) PD edge cases (2 GPUs) + key: multiconnector-nixl-offloading-pd-edge-cases-2-gpus timeout_in_minutes: 30 working_dir: "/vllm-workspace/tests" num_devices: 2 diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml index 4734b9d3e1..6f45a38eeb 100644 --- a/.buildkite/test_areas/distributed.yaml +++ b/.buildkite/test_areas/distributed.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: Distributed Comm Ops + key: distributed-comm-ops timeout_in_minutes: 20 working_dir: "/vllm-workspace/tests" num_devices: 2 @@ -16,6 +17,7 @@ steps: - pytest -v -s distributed/test_shm_storage.py - label: Distributed DP Tests (2 GPUs) + key: distributed-dp-tests-2-gpus timeout_in_minutes: 20 working_dir: "/vllm-workspace/tests" num_devices: 2 @@ -37,6 +39,7 @@ steps: - DP_SIZE=2 pytest -v -s entrypoints/openai/test_multi_api_servers.py - label: Distributed Compile + RPC Tests (2 GPUs) + key: distributed-compile-rpc-tests-2-gpus timeout_in_minutes: 20 working_dir: "/vllm-workspace/tests" num_devices: 2 @@ -59,6 +62,7 @@ steps: - pytest -v -s ./compile/test_wrapper.py - label: Distributed Torchrun + Shutdown Tests (2 GPUs) + key: distributed-torchrun-shutdown-tests-2-gpus timeout_in_minutes: 20 working_dir: "/vllm-workspace/tests" num_devices: 2 @@ -81,6 +85,7 @@ steps: - pytest -v -s v1/worker/test_worker_memory_snapshot.py - label: Distributed Torchrun + Examples (4 GPUs) + key: distributed-torchrun-examples-4-gpus timeout_in_minutes: 30 working_dir: "/vllm-workspace" num_devices: 4 @@ -112,6 +117,7 @@ steps: - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_ipc.py - label: Distributed DP Tests (4 GPUs) + key: distributed-dp-tests-4-gpus timeout_in_minutes: 30 working_dir: "/vllm-workspace/tests" num_devices: 4 @@ -132,6 +138,7 @@ steps: - pytest -v -s distributed/test_utils.py - label: Distributed Compile + Comm (4 GPUs) + key: distributed-compile-comm-4-gpus timeout_in_minutes: 30 working_dir: "/vllm-workspace/tests" num_devices: 4 @@ -153,6 +160,7 @@ steps: - pytest -v -s distributed/test_multiproc_executor.py::test_multiproc_executor_multi_node - label: Distributed Tests (8 GPUs)(H100) + key: distributed-tests-8-gpus-h100 timeout_in_minutes: 10 device: h100 num_devices: 8 @@ -173,6 +181,7 @@ steps: - torchrun --nproc-per-node=8 ../examples/features/torchrun/torchrun_dp_example_offline.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep - label: Distributed Tests (4 GPUs)(A100) + key: distributed-tests-4-gpus-a100 device: a100 optional: true num_devices: 4 @@ -187,6 +196,7 @@ steps: - pytest -v -s -x lora/test_mixtral.py - label: Distributed Tests (2 GPUs)(H100) + key: distributed-tests-2-gpus-h100 timeout_in_minutes: 15 device: h100 optional: true @@ -201,6 +211,7 @@ steps: - pytest -v -s tests/distributed/test_packed_tensor.py - label: Distributed Tests (2 GPUs)(B200) + key: distributed-tests-2-gpus-b200 device: b200 optional: true working_dir: "/vllm-workspace/" @@ -214,6 +225,7 @@ steps: - label: 2 Node Test (4 GPUs) + key: 2-node-test-4-gpus timeout_in_minutes: 30 working_dir: "/vllm-workspace/tests" num_devices: 2 @@ -231,6 +243,7 @@ steps: - ./.buildkite/scripts/run-multi-node-test.sh /vllm-workspace/tests 2 2 $IMAGE_TAG "VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed' && NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed' && python3 ../examples/features/data_parallel/data_parallel_offline.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=0 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code && VLLM_MULTI_NODE=1 pytest -v -s distributed/test_multi_node_assignment.py && VLLM_MULTI_NODE=1 pytest -v -s distributed/test_pipeline_parallel.py" "VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed' && NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed' && python3 ../examples/features/data_parallel/data_parallel_offline.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=1 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code" - label: Pipeline + Context Parallelism (4 GPUs) + key: pipeline-context-parallelism-4-gpus timeout_in_minutes: 60 working_dir: "/vllm-workspace/tests" num_devices: 4 @@ -245,6 +258,7 @@ steps: - pytest -v -s distributed/test_pipeline_parallel.py - label: RayExecutorV2 (4 GPUs) + key: rayexecutorv2-4-gpus timeout_in_minutes: 60 working_dir: "/vllm-workspace/tests" num_devices: 4 diff --git a/.buildkite/test_areas/e2e_integration.yaml b/.buildkite/test_areas/e2e_integration.yaml index 857fefd268..8df3b2fd0d 100644 --- a/.buildkite/test_areas/e2e_integration.yaml +++ b/.buildkite/test_areas/e2e_integration.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: DeepSeek V2-Lite Accuracy + key: deepseek-v2-lite-accuracy timeout_in_minutes: 60 device: h100 optional: true @@ -12,6 +13,7 @@ steps: - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010 - label: Qwen3-30B-A3B-FP8-block Accuracy + key: qwen3-30b-a3b-fp8-block-accuracy timeout_in_minutes: 60 device: h100 optional: true @@ -21,6 +23,7 @@ steps: - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 - label: Qwen3-30B-A3B-FP8-block Accuracy (B200) + key: qwen3-30b-a3b-fp8-block-accuracy-b200 timeout_in_minutes: 60 device: b200 optional: true @@ -30,6 +33,7 @@ steps: - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 2 1 - label: Qwen3-30B-A3B-FP8 DP4 Async EPLB Accuracy + key: qwen3-30b-a3b-fp8-dp4-async-eplb-accuracy timeout_in_minutes: 60 device: h100 optional: true @@ -39,6 +43,7 @@ steps: - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_dp4_async_eplb.sh 0.8 200 8050 - label: DeepSeek V2-Lite Prefetch Offload Accuracy (H100) + key: deepseek-v2-lite-prefetch-offload-accuracy-h100 timeout_in_minutes: 60 device: h100 optional: true diff --git a/.buildkite/test_areas/engine.yaml b/.buildkite/test_areas/engine.yaml index 5e4361ec9a..17ac44803a 100644 --- a/.buildkite/test_areas/engine.yaml +++ b/.buildkite/test_areas/engine.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: Engine + key: engine timeout_in_minutes: 15 device: h200_18gb source_file_dependencies: @@ -16,6 +17,7 @@ steps: - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py - label: Engine (1 GPU) + key: engine-1-gpu timeout_in_minutes: 30 source_file_dependencies: - vllm/v1/engine/ @@ -25,6 +27,7 @@ steps: - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py - label: e2e Scheduling (1 GPU) + key: e2e-scheduling-1-gpu timeout_in_minutes: 30 device: h200_18gb source_file_dependencies: @@ -34,6 +37,7 @@ steps: - pytest -v -s v1/e2e/general/test_async_scheduling.py - label: e2e Core (1 GPU) + key: e2e-core-1-gpu timeout_in_minutes: 30 source_file_dependencies: - vllm/v1/ @@ -42,6 +46,7 @@ steps: - pytest -v -s v1/e2e/general --ignore v1/e2e/general/test_async_scheduling.py - label: V1 e2e (2 GPUs) + key: v1-e2e-2-gpus timeout_in_minutes: 60 # TODO: Fix timeout after we have more confidence in the test stability optional: true num_devices: 2 @@ -58,6 +63,7 @@ steps: - image-build-amd - label: V1 e2e (4 GPUs) + key: v1-e2e-4-gpus timeout_in_minutes: 60 # TODO: Fix timeout after we have more confidence in the test stability optional: true num_devices: 4 @@ -74,6 +80,7 @@ steps: - image-build-amd - label: V1 e2e (4xH100) + key: v1-e2e-4xh100 timeout_in_minutes: 60 device: h100 num_devices: 4 diff --git a/.buildkite/test_areas/entrypoints.yaml b/.buildkite/test_areas/entrypoints.yaml index 8c2b529a80..4908df1e4e 100644 --- a/.buildkite/test_areas/entrypoints.yaml +++ b/.buildkite/test_areas/entrypoints.yaml @@ -2,7 +2,8 @@ group: Entrypoints depends_on: - image-build steps: -- label: Entrypoints Unit Tests +- label: Entrypoints Unit Tests + key: entrypoints-unit-tests timeout_in_minutes: 10 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -13,6 +14,7 @@ steps: - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/serve/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling - label: Entrypoints Integration (LLM) + key: entrypoints-integration-llm timeout_in_minutes: 40 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -26,6 +28,7 @@ steps: - pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests - label: Entrypoints Integration (API Server openai - Part 1) + key: entrypoints-integration-api-server-openai-part-1 timeout_in_minutes: 50 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -43,6 +46,7 @@ steps: - label: Entrypoints Integration (API Server openai - Part 2) + key: entrypoints-integration-api-server-openai-part-2 timeout_in_minutes: 50 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -60,6 +64,7 @@ steps: - image-build-amd - label: Entrypoints Integration (API Server openai - Part 3) + key: entrypoints-integration-api-server-openai-part-3 timeout_in_minutes: 50 device: h200_18gb working_dir: "/vllm-workspace/tests" @@ -72,6 +77,7 @@ steps: - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/speech_to_text/ --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/test_multi_api_servers.py - label: Entrypoints Integration (API Server 2) + key: entrypoints-integration-api-server-2 timeout_in_minutes: 130 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -86,6 +92,7 @@ steps: - pytest -v -s tool_use - label: Entrypoints Integration (Pooling) + key: entrypoints-integration-pooling timeout_in_minutes: 50 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -96,6 +103,7 @@ steps: - pytest -v -s entrypoints/pooling - label: Entrypoints Integration (Responses API) + key: entrypoints-integration-responses-api timeout_in_minutes: 50 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -105,6 +113,7 @@ steps: - pytest -v -s entrypoints/openai/responses - label: OpenAI API Correctness + key: openai-api-correctness timeout_in_minutes: 30 device: h200_18gb source_file_dependencies: diff --git a/.buildkite/test_areas/expert_parallelism.yaml b/.buildkite/test_areas/expert_parallelism.yaml index c2adf52a2d..0f7ab0d715 100644 --- a/.buildkite/test_areas/expert_parallelism.yaml +++ b/.buildkite/test_areas/expert_parallelism.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: EPLB Algorithm + key: eplb-algorithm timeout_in_minutes: 15 device: h200_18gb working_dir: "/vllm-workspace/tests" @@ -15,6 +16,7 @@ steps: - pytest -v -s distributed/test_eplb_utils.py - label: EPLB Execution # 17min + key: eplb-execution timeout_in_minutes: 27 working_dir: "/vllm-workspace/tests" num_devices: 4 @@ -26,6 +28,7 @@ steps: - pytest -v -s distributed/test_eplb_spec_decode.py - label: Elastic EP Scaling Test + key: elastic-ep-scaling-test timeout_in_minutes: 20 device: h100 working_dir: "/vllm-workspace/tests" diff --git a/.buildkite/test_areas/kernels.yaml b/.buildkite/test_areas/kernels.yaml index 86e09f3de4..27451dd1bc 100644 --- a/.buildkite/test_areas/kernels.yaml +++ b/.buildkite/test_areas/kernels.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: vLLM IR Tests + key: vllm-ir-tests timeout_in_minutes: 10 device: h200_18gb working_dir: "/vllm-workspace/" @@ -14,6 +15,7 @@ steps: - pytest -v -s tests/kernels/ir - label: Kernels Core Operation Test + key: kernels-core-operation-test timeout_in_minutes: 75 source_file_dependencies: - csrc/ @@ -23,6 +25,7 @@ steps: - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_concat_mla_q.py - label: Kernels MiniMax Reduce RMS Test (2 GPUs) + key: kernels-minimax-reduce-rms-test-2-gpus timeout_in_minutes: 15 num_devices: 2 device: h100 @@ -36,6 +39,7 @@ steps: - pytest -v -s kernels/core/test_minimax_reduce_rms.py - label: Kernels Attention Test %N + key: kernels-attention-test timeout_in_minutes: 35 source_file_dependencies: - csrc/attention/ @@ -49,6 +53,7 @@ steps: parallelism: 2 - label: Kernels Quantization Test %N + key: kernels-quantization-test timeout_in_minutes: 90 source_file_dependencies: - csrc/quantization/ @@ -59,6 +64,7 @@ steps: parallelism: 2 - label: Kernels MoE Test %N + key: kernels-moe-test timeout_in_minutes: 25 source_file_dependencies: - csrc/quantization/cutlass_w8a8/moe/ @@ -74,6 +80,7 @@ steps: parallelism: 5 - label: Kernels Mamba Test + key: kernels-mamba-test timeout_in_minutes: 45 source_file_dependencies: - csrc/mamba/ @@ -83,6 +90,7 @@ steps: - pytest -v -s kernels/mamba - label: Kernels DeepGEMM Test (H100) + key: kernels-deepgemm-test-h100 timeout_in_minutes: 45 device: h100 num_devices: 1 @@ -104,6 +112,7 @@ steps: - pytest -v -s quantization/test_cutlass_w4a16.py - label: Kernels (B200) + key: kernels-b200 timeout_in_minutes: 30 working_dir: "/vllm-workspace/" device: b200 @@ -152,6 +161,7 @@ steps: - pytest -v -s tests/models/quantization/test_nvfp4.py - label: Kernels Helion Test + key: kernels-helion-test timeout_in_minutes: 30 device: h100 source_file_dependencies: @@ -163,6 +173,7 @@ steps: - label: Kernels FP8 MoE Test (1 H100) + key: kernels-fp8-moe-test-1-h100 timeout_in_minutes: 90 device: h100 num_devices: 1 @@ -179,6 +190,7 @@ steps: - pytest -v -s kernels/moe/test_triton_moe_ptpc_fp8.py - label: Kernels FP8 MoE Test (2 H100s) + key: kernels-fp8-moe-test-2-h100s timeout_in_minutes: 90 device: h100 num_devices: 2 @@ -188,6 +200,7 @@ steps: - pytest -v -s kernels/moe/test_deepep_moe.py - label: Kernels Fp4 MoE Test (B200) + key: kernels-fp4-moe-test-b200 timeout_in_minutes: 60 device: b200 num_devices: 1 @@ -200,6 +213,7 @@ steps: - label: Kernels FusedMoE Layer Test (2 H100s) + key: kernels-fusedmoe-layer-test-2-h100s timeout_in_minutes: 90 device: h100 num_devices: 2 @@ -216,6 +230,7 @@ steps: - label: Kernels FusedMoE Layer Test (2 B200s) + key: kernels-fusedmoe-layer-test-2-b200s timeout_in_minutes: 90 device: b200 num_devices: 2 diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index a07d702cf3..38b64b163e 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: LM Eval Small Models + key: lm-eval-small-models timeout_in_minutes: 75 source_file_dependencies: - csrc/ @@ -24,6 +25,7 @@ steps: # - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4 - label: LM Eval Large Models (4 GPUs)(H100) + key: lm-eval-large-models-4-gpus-h100 device: h100 optional: true num_devices: 4 @@ -36,6 +38,7 @@ steps: - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-hopper.txt --tp-size=4 - label: LM Eval Small Models (B200) + key: lm-eval-small-models-b200 timeout_in_minutes: 120 device: b200 optional: true @@ -46,6 +49,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt - label: LM Eval Qwen3.5 Models (B200) + key: lm-eval-qwen3-5-models-b200 timeout_in_minutes: 120 device: b200 optional: true @@ -62,6 +66,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-blackwell.txt - label: LM Eval Large Models (H200) + key: lm-eval-large-models-h200 timeout_in_minutes: 60 device: h200 optional: true @@ -70,6 +75,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200.txt - label: MoE Refactor Integration Test (H100 - TEMPORARY) + key: moe-refactor-integration-test-h100-temporary device: h100 optional: true num_devices: 2 @@ -77,6 +83,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor/config-h100.txt - label: MoE Refactor Integration Test (B200 - TEMPORARY) + key: moe-refactor-integration-test-b200-temporary device: b200 optional: true num_devices: 2 @@ -84,6 +91,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor/config-b200.txt - label: MoE Refactor Integration Test (B200 DP - TEMPORARY) + key: moe-refactor-integration-test-b200-dp-temporary device: b200 optional: true num_devices: 2 @@ -92,6 +100,7 @@ steps: - label: LM Eval TurboQuant KV Cache + key: lm-eval-turboquant-kv-cache timeout_in_minutes: 75 source_file_dependencies: - vllm/model_executor/layers/quantization/turboquant/ @@ -102,6 +111,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant.txt - label: GPQA Eval (GPT-OSS) (H100) + key: gpqa-eval-gpt-oss-h100 timeout_in_minutes: 120 device: h100 optional: true @@ -115,6 +125,7 @@ steps: - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-h100.txt - label: GPQA Eval (GPT-OSS) (B200) + key: gpqa-eval-gpt-oss-b200 timeout_in_minutes: 120 device: b200 optional: true diff --git a/.buildkite/test_areas/lora.yaml b/.buildkite/test_areas/lora.yaml index 21f392ff73..f540eb2fcc 100644 --- a/.buildkite/test_areas/lora.yaml +++ b/.buildkite/test_areas/lora.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: LoRA %N + key: lora timeout_in_minutes: 30 source_file_dependencies: - vllm/lora @@ -13,6 +14,7 @@ steps: - label: LoRA TP (Distributed) + key: lora-tp-distributed timeout_in_minutes: 30 num_devices: 4 source_file_dependencies: diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index 1552aceab4..3ff96dcbdf 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: V1 Spec Decode + key: v1-spec-decode timeout_in_minutes: 30 source_file_dependencies: - vllm/ @@ -18,6 +19,7 @@ steps: - image-build-amd - label: V1 Sample + Logits + key: v1-sample-logits timeout_in_minutes: 30 device: h200_18gb source_file_dependencies: @@ -41,6 +43,7 @@ steps: - image-build-amd - label: V1 Core + KV + Metrics + key: v1-core-kv-metrics timeout_in_minutes: 30 source_file_dependencies: - vllm/ @@ -71,6 +74,7 @@ steps: - image-build-amd - label: V1 Others (CPU) + key: v1-others-cpu depends_on: - image-build-cpu source_file_dependencies: @@ -86,6 +90,7 @@ steps: - pytest -v -s -m 'cpu_test' v1/metrics - label: Regression + key: regression timeout_in_minutes: 20 device: h200_18gb source_file_dependencies: @@ -97,6 +102,7 @@ steps: working_dir: "/vllm-workspace/tests" # optional - label: Examples + key: examples timeout_in_minutes: 45 working_dir: "/vllm-workspace/examples" source_file_dependencies: @@ -128,6 +134,7 @@ steps: - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 - label: Metrics, Tracing (2 GPUs) + key: metrics-tracing-2-gpus timeout_in_minutes: 20 num_devices: 2 source_file_dependencies: @@ -142,6 +149,7 @@ steps: - pytest -v -s v1/tracing - label: Python-only Installation + key: python-only-installation depends_on: ~ timeout_in_minutes: 20 source_file_dependencies: @@ -151,6 +159,7 @@ steps: - bash standalone_tests/python_only_compile.sh - label: Async Engine, Inputs, Utils, Worker + key: async-engine-inputs-utils-worker timeout_in_minutes: 50 source_file_dependencies: - vllm/ @@ -163,7 +172,8 @@ steps: - pytest -v -s utils_ - label: Async Engine, Inputs, Utils, Worker, Config (CPU) - depends_on: + key: async-engine-inputs-utils-worker-config-cpu + depends_on: - image-build-cpu timeout_in_minutes: 30 source_file_dependencies: @@ -196,6 +206,7 @@ steps: - pytest -v -s config - label: Batch Invariance (H100) + key: batch-invariance-h100 timeout_in_minutes: 30 device: h100 source_file_dependencies: @@ -211,6 +222,7 @@ steps: - VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[FLASH_ATTN] - label: Batch Invariance (B200) + key: batch-invariance-b200 timeout_in_minutes: 30 device: b200 source_file_dependencies: @@ -227,6 +239,7 @@ steps: - pytest -v -s v1/determinism/test_nvfp4_batch_invariant.py - label: Acceptance Length Test (Large Models) # optional + key: acceptance-length-test-large-models timeout_in_minutes: 25 gpu: h100 optional: true diff --git a/.buildkite/test_areas/model_executor.yaml b/.buildkite/test_areas/model_executor.yaml index 212abfdbb9..c41ef8a711 100644 --- a/.buildkite/test_areas/model_executor.yaml +++ b/.buildkite/test_areas/model_executor.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: Model Executor + key: model-executor timeout_in_minutes: 35 source_file_dependencies: - vllm/engine/arg_utils.py diff --git a/.buildkite/test_areas/model_runner_v2.yaml b/.buildkite/test_areas/model_runner_v2.yaml index 74025d34f8..6a4338a5e4 100644 --- a/.buildkite/test_areas/model_runner_v2.yaml +++ b/.buildkite/test_areas/model_runner_v2.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: Model Runner V2 Core Tests + key: model-runner-v2-core-tests timeout_in_minutes: 45 source_file_dependencies: - vllm/v1/worker/gpu/ @@ -25,6 +26,7 @@ steps: - pytest -v -s entrypoints/llm/test_struct_output_generate.py -k "xgrammar and not speculative_config6 and not speculative_config7 and not speculative_config8 and not speculative_config0" - label: Model Runner V2 Examples + key: model-runner-v2-examples timeout_in_minutes: 45 working_dir: "/vllm-workspace/examples" source_file_dependencies: @@ -60,6 +62,7 @@ steps: - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 - label: Model Runner V2 Distributed (2 GPUs) + key: model-runner-v2-distributed-2-gpus timeout_in_minutes: 45 working_dir: "/vllm-workspace/tests" num_devices: 2 @@ -80,6 +83,7 @@ steps: - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py - label: Model Runner V2 Pipeline Parallelism (4 GPUs) + key: model-runner-v2-pipeline-parallelism-4-gpus timeout_in_minutes: 60 working_dir: "/vllm-workspace/tests" num_devices: 4 @@ -95,6 +99,7 @@ steps: - pytest -v -s distributed/test_pp_cudagraph.py -k "not ray" - label: Model Runner V2 Spec Decode + key: model-runner-v2-spec-decode timeout_in_minutes: 30 working_dir: "/vllm-workspace/tests" source_file_dependencies: diff --git a/.buildkite/test_areas/models_basic.yaml b/.buildkite/test_areas/models_basic.yaml index 73cf8c53bc..ab597f960e 100644 --- a/.buildkite/test_areas/models_basic.yaml +++ b/.buildkite/test_areas/models_basic.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: Basic Models Tests (Initialization) + key: basic-models-tests-initialization timeout_in_minutes: 45 torch_nightly: true source_file_dependencies: @@ -16,6 +17,7 @@ steps: torch_nightly: {} - label: Basic Models Tests (Extra Initialization) %N + key: basic-models-tests-extra-initialization timeout_in_minutes: 45 source_file_dependencies: - vllm/model_executor/models/ @@ -31,6 +33,7 @@ steps: torch_nightly: {} - label: Basic Models Tests (Other) + key: basic-models-tests-other timeout_in_minutes: 45 source_file_dependencies: - vllm/ @@ -47,6 +50,7 @@ steps: - label: Basic Models Test (Other CPU) # 5min + key: basic-models-test-other-cpu depends_on: - image-build-cpu timeout_in_minutes: 10 @@ -59,6 +63,7 @@ steps: - pytest -v -s models/test_utils.py models/test_vision.py - label: Transformers Nightly Models + key: transformers-nightly-models working_dir: "/vllm-workspace/" optional: true soft_fail: true @@ -74,6 +79,7 @@ steps: - VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/generate/multimodal/audio_language_offline.py --model-type whisper - label: Transformers Backward Compatibility Models Test + key: transformers-backward-compatibility-models-test working_dir: "/vllm-workspace/" optional: true soft_fail: true diff --git a/.buildkite/test_areas/models_distributed.yaml b/.buildkite/test_areas/models_distributed.yaml index 55e7410b8a..b5758c55af 100644 --- a/.buildkite/test_areas/models_distributed.yaml +++ b/.buildkite/test_areas/models_distributed.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: Distributed Model Tests (2 GPUs) + key: distributed-model-tests-2-gpus timeout_in_minutes: 50 working_dir: "/vllm-workspace/tests" num_devices: 2 diff --git a/.buildkite/test_areas/models_language.yaml b/.buildkite/test_areas/models_language.yaml index c13371e25f..64f8fc809d 100644 --- a/.buildkite/test_areas/models_language.yaml +++ b/.buildkite/test_areas/models_language.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: Language Models Tests (Standard) + key: language-models-tests-standard timeout_in_minutes: 25 source_file_dependencies: - vllm/ @@ -15,6 +16,7 @@ steps: torch_nightly: {} - label: Language Models Tests (Extra Standard) %N + key: language-models-tests-extra-standard timeout_in_minutes: 45 source_file_dependencies: - vllm/model_executor/models/ @@ -31,6 +33,7 @@ steps: torch_nightly: {} - label: Language Models Tests (Hybrid) %N + key: language-models-tests-hybrid timeout_in_minutes: 75 source_file_dependencies: - vllm/ @@ -47,6 +50,7 @@ steps: torch_nightly: {} - label: Language Models Test (Extended Generation) # 80min + key: language-models-test-extended-generation timeout_in_minutes: 110 optional: true source_file_dependencies: @@ -69,6 +73,7 @@ steps: - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)' - label: Language Models Test (PPL) + key: language-models-test-ppl timeout_in_minutes: 110 device: h200_18gb optional: true @@ -79,6 +84,7 @@ steps: - pytest -v -s models/language/generation_ppl_test - label: Language Models Test (Extended Pooling) # 36min + key: language-models-test-extended-pooling timeout_in_minutes: 50 optional: true source_file_dependencies: @@ -93,6 +99,7 @@ steps: - image-build-amd - label: Language Models Test (MTEB) + key: language-models-test-mteb timeout_in_minutes: 110 device: h200_18gb optional: true diff --git a/.buildkite/test_areas/models_multimodal.yaml b/.buildkite/test_areas/models_multimodal.yaml index 245ef24026..857cc68f2a 100644 --- a/.buildkite/test_areas/models_multimodal.yaml +++ b/.buildkite/test_areas/models_multimodal.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: "Multi-Modal Models (Standard) 1: qwen2" + key: multi-modal-models-standard-1-qwen2 timeout_in_minutes: 45 device: h200_18gb source_file_dependencies: @@ -19,6 +20,7 @@ steps: - image-build-amd - label: "Multi-Modal Models (Standard) 2: qwen3 + gemma" + key: multi-modal-models-standard-2-qwen3-gemma timeout_in_minutes: 45 device: h200_18gb source_file_dependencies: @@ -36,6 +38,7 @@ steps: - image-build-amd - label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl" + key: multi-modal-models-standard-3-llava-qwen2-vl timeout_in_minutes: 45 source_file_dependencies: - vllm/ @@ -51,6 +54,7 @@ steps: - image-build-amd - label: "Multi-Modal Models (Standard) 4: other + whisper" + key: multi-modal-models-standard-4-other-whisper timeout_in_minutes: 45 source_file_dependencies: - vllm/ @@ -67,7 +71,8 @@ steps: - image-build-amd - label: Multi-Modal Processor (CPU) - depends_on: + key: multi-modal-processor-cpu + depends_on: - image-build-cpu timeout_in_minutes: 60 source_file_dependencies: @@ -80,6 +85,7 @@ steps: - pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py - label: Multi-Modal Processor # 44min + key: multi-modal-processor timeout_in_minutes: 60 device: h200_18gb source_file_dependencies: @@ -91,6 +97,7 @@ steps: - pytest -v -s models/multimodal/processing/test_tensor_schema.py - label: Multi-Modal Accuracy Eval (Small Models) # 50min + key: multi-modal-accuracy-eval-small-models timeout_in_minutes: 70 working_dir: "/vllm-workspace/.buildkite/lm-eval-harness" source_file_dependencies: @@ -101,6 +108,7 @@ steps: - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1 - label: Multi-Modal Models (Extended Generation 1) + key: multi-modal-models-extended-generation-1 optional: true source_file_dependencies: - vllm/ @@ -117,6 +125,7 @@ steps: - image-build-amd - label: Multi-Modal Models (Extended Generation 2) + key: multi-modal-models-extended-generation-2 optional: true source_file_dependencies: - vllm/ @@ -126,6 +135,7 @@ steps: - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model' - label: Multi-Modal Models (Extended Generation 3) + key: multi-modal-models-extended-generation-3 optional: true source_file_dependencies: - vllm/ @@ -135,6 +145,7 @@ steps: - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model' - label: Multi-Modal Models (Extended Pooling) + key: multi-modal-models-extended-pooling optional: true device: h200_18gb source_file_dependencies: diff --git a/.buildkite/test_areas/plugins.yaml b/.buildkite/test_areas/plugins.yaml index 8e0eb02840..0d23180f3e 100644 --- a/.buildkite/test_areas/plugins.yaml +++ b/.buildkite/test_areas/plugins.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: Plugin Tests (2 GPUs) + key: plugin-tests-2-gpus timeout_in_minutes: 60 working_dir: "/vllm-workspace/tests" num_devices: 2 diff --git a/.buildkite/test_areas/pytorch.yaml b/.buildkite/test_areas/pytorch.yaml index a3648219d8..a470cc60a3 100644 --- a/.buildkite/test_areas/pytorch.yaml +++ b/.buildkite/test_areas/pytorch.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: PyTorch Compilation Unit Tests + key: pytorch-compilation-unit-tests timeout_in_minutes: 10 source_file_dependencies: - vllm/ @@ -18,6 +19,7 @@ steps: - "find compile/ -maxdepth 1 -name 'test_*.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'" - label: PyTorch Compilation Unit Tests (H100) + key: pytorch-compilation-unit-tests-h100 timeout_in_minutes: 30 device: h100 num_devices: 1 @@ -28,6 +30,7 @@ steps: - "find compile/h100/ -name 'test_*.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'" - label: PyTorch Compilation Passes Unit Tests + key: pytorch-compilation-passes-unit-tests timeout_in_minutes: 20 source_file_dependencies: - vllm/ @@ -36,6 +39,7 @@ steps: - pytest -s -v compile/passes --ignore compile/passes/distributed - label: PyTorch Fullgraph Smoke Test + key: pytorch-fullgraph-smoke-test timeout_in_minutes: 35 source_file_dependencies: - vllm/ @@ -48,6 +52,7 @@ steps: - "find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_graph.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'" - label: PyTorch Fullgraph + key: pytorch-fullgraph timeout_in_minutes: 30 device: h200_18gb source_file_dependencies: @@ -58,6 +63,7 @@ steps: - pytest -v -s compile/fullgraph/test_full_graph.py -k 'not test_fp8_kv_scale_compile' - label: Pytorch Nightly Dependency Override Check # 2min + key: pytorch-nightly-dependency-override-check # if this test fails, it means the nightly torch version is not compatible with some # of the dependencies. Please check the error message and add the package to whitelist # in /vllm/tools/pre_commit/generate_nightly_torch_test.py diff --git a/.buildkite/test_areas/quantization.yaml b/.buildkite/test_areas/quantization.yaml index a42d59b021..0ccca52840 100644 --- a/.buildkite/test_areas/quantization.yaml +++ b/.buildkite/test_areas/quantization.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: Quantization + key: quantization timeout_in_minutes: 90 source_file_dependencies: - csrc/ @@ -21,6 +22,7 @@ steps: - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py - label: Quantized MoE Test (B200) + key: quantized-moe-test-b200 timeout_in_minutes: 60 working_dir: "/vllm-workspace/" device: b200 @@ -38,6 +40,7 @@ steps: - pytest -s -v tests/quantization/test_blackwell_moe.py - label: Quantized Models Test + key: quantized-models-test timeout_in_minutes: 60 source_file_dependencies: - vllm/model_executor/layers/quantization diff --git a/.buildkite/test_areas/ray_compat.yaml b/.buildkite/test_areas/ray_compat.yaml index 3485e34653..9207621a58 100644 --- a/.buildkite/test_areas/ray_compat.yaml +++ b/.buildkite/test_areas/ray_compat.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: Ray Dependency Compatibility Check + key: ray-dependency-compatibility-check # Informational only — does not block the pipeline. # If this fails, it means the PR introduces a dependency that # conflicts with Ray's dependency constraints. diff --git a/.buildkite/test_areas/samplers.yaml b/.buildkite/test_areas/samplers.yaml index 6fafda70bd..37f8eaa688 100644 --- a/.buildkite/test_areas/samplers.yaml +++ b/.buildkite/test_areas/samplers.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: Samplers Test + key: samplers-test timeout_in_minutes: 75 source_file_dependencies: - vllm/model_executor/layers diff --git a/.buildkite/test_areas/spec_decode.yaml b/.buildkite/test_areas/spec_decode.yaml index 05925da0da..a92df3ed58 100644 --- a/.buildkite/test_areas/spec_decode.yaml +++ b/.buildkite/test_areas/spec_decode.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: Spec Decode Eagle + key: spec-decode-eagle timeout_in_minutes: 30 device: h200_18gb source_file_dependencies: @@ -13,6 +14,7 @@ steps: - pytest -v -s v1/e2e/spec_decode -k "eagle_correctness" - label: Spec Decode Eagle Nightly B200 + key: spec-decode-eagle-nightly-b200 timeout_in_minutes: 30 device: b200 optional: true @@ -24,6 +26,7 @@ steps: - pytest -v -s v1/e2e/spec_decode -k "eagle_correctness" - label: Spec Decode Speculators + MTP + key: spec-decode-speculators-mtp timeout_in_minutes: 30 device: h200_18gb source_file_dependencies: @@ -35,6 +38,7 @@ steps: - pytest -v -s v1/e2e/spec_decode -k "speculators or mtp_correctness" - label: Spec Decode Speculators + MTP Nightly B200 + key: spec-decode-speculators-mtp-nightly-b200 timeout_in_minutes: 30 device: b200 optional: true @@ -47,6 +51,7 @@ steps: - pytest -v -s v1/e2e/spec_decode -k "speculators or mtp_correctness" - label: Spec Decode Ngram + Suffix + key: spec-decode-ngram-suffix timeout_in_minutes: 30 device: h200_18gb source_file_dependencies: @@ -57,6 +62,7 @@ steps: - pytest -v -s v1/e2e/spec_decode -k "ngram or suffix" - label: Spec Decode Draft Model + key: spec-decode-draft-model timeout_in_minutes: 30 device: h200_18gb source_file_dependencies: @@ -67,6 +73,7 @@ steps: - pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference" - label: Spec Decode Draft Model Nightly B200 + key: spec-decode-draft-model-nightly-b200 timeout_in_minutes: 30 device: b200 optional: true @@ -78,6 +85,7 @@ steps: - pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference" - label: DFlash Speculators Correctness + key: dflash-speculators-correctness timeout_in_minutes: 30 device: h100 optional: true diff --git a/.buildkite/test_areas/weight_loading.yaml b/.buildkite/test_areas/weight_loading.yaml index 8e86374a8a..01c6bb7809 100644 --- a/.buildkite/test_areas/weight_loading.yaml +++ b/.buildkite/test_areas/weight_loading.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: Weight Loading Multiple GPU # 33min + key: weight-loading-multiple-gpu timeout_in_minutes: 45 working_dir: "/vllm-workspace/tests" num_devices: 2