From 8d6b18329a303c3d4a8f1a7b515cd69a4ef64800 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Tue, 18 Aug 2026 12:50:32 -0700 Subject: [PATCH] [CI] Standardize test job labels by device (#52659) Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com> Co-authored-by: OpenAI Codex --- .buildkite/test_areas/attention.yaml | 5 +- .buildkite/test_areas/basic_correctness.yaml | 3 +- .buildkite/test_areas/benchmarks.yaml | 5 +- .buildkite/test_areas/compile.yaml | 22 ++++---- .buildkite/test_areas/cuda.yaml | 4 +- .buildkite/test_areas/disaggregated.yaml | 31 ++++++----- .../test_areas/disaggregated_mooncake.yaml | 2 +- .buildkite/test_areas/distributed.yaml | 29 +++++----- .buildkite/test_areas/docker.yaml | 2 +- .buildkite/test_areas/e2e_integration.yaml | 10 ++-- .buildkite/test_areas/engine.yaml | 17 +++--- .buildkite/test_areas/entrypoints.yaml | 28 ++++++---- .buildkite/test_areas/expert_parallelism.yaml | 7 +-- .buildkite/test_areas/fault_tolerance.yaml | 2 +- .buildkite/test_areas/jit_monitor.yaml | 2 +- .buildkite/test_areas/kernels.yaml | 45 ++++++++-------- .buildkite/test_areas/lm_eval.yaml | 54 ++++++++++--------- .buildkite/test_areas/lora.yaml | 5 +- .buildkite/test_areas/misc.yaml | 35 ++++++------ .buildkite/test_areas/model_executor.yaml | 3 +- .buildkite/test_areas/model_runner_v2.yaml | 10 ++-- .buildkite/test_areas/models_basic.yaml | 13 ++--- .buildkite/test_areas/models_distributed.yaml | 2 +- .buildkite/test_areas/models_language.yaml | 20 ++++--- .buildkite/test_areas/models_multimodal.yaml | 32 ++++++----- .buildkite/test_areas/plugins.yaml | 8 +-- .buildkite/test_areas/pytorch.yaml | 13 ++--- .buildkite/test_areas/quantization.yaml | 8 +-- .buildkite/test_areas/ray_compat.yaml | 2 +- .buildkite/test_areas/rust_frontend.yaml | 10 ++-- .../test_areas/rust_frontend_cargo.yaml | 4 +- .buildkite/test_areas/samplers.yaml | 3 +- .buildkite/test_areas/spec_decode.yaml | 37 ++++++++----- .buildkite/test_areas/torch_abi.yaml | 2 +- .buildkite/test_areas/weight_loading.yaml | 3 +- 35 files changed, 268 insertions(+), 210 deletions(-) diff --git a/.buildkite/test_areas/attention.yaml b/.buildkite/test_areas/attention.yaml index 71d2975a5f..e11108d925 100644 --- a/.buildkite/test_areas/attention.yaml +++ b/.buildkite/test_areas/attention.yaml @@ -2,7 +2,7 @@ group: Attention depends_on: - image-build steps: -- label: V1 attention (H100-MI300) +- label: ":nvidia: (H100) V1 Attention Shard %N" key: v1-attention-h100-mi300 timeout_in_minutes: 85 device: h100 @@ -16,6 +16,7 @@ steps: parallelism: 2 mirror: amd: + label: ":amd: (MI300) V1 Attention Shard %N" dind: false device: mi300_1 timeout_in_minutes: 125 @@ -30,7 +31,7 @@ steps: - vllm/envs.py - vllm/platforms/rocm.py -- label: V1 attention (B200) +- label: ":nvidia: (B200) V1 Attention Shard %N" key: v1-attention-b200 timeout_in_minutes: 80 device: b200-k8s diff --git a/.buildkite/test_areas/basic_correctness.yaml b/.buildkite/test_areas/basic_correctness.yaml index d0dac3a112..1ce9dcbe88 100644 --- a/.buildkite/test_areas/basic_correctness.yaml +++ b/.buildkite/test_areas/basic_correctness.yaml @@ -2,7 +2,7 @@ group: Basic Correctness depends_on: - image-build steps: -- label: Basic Correctness +- label: ":nvidia: (H200) Basic Correctness" key: basic-correctness timeout_in_minutes: 68 device: h200_18gb @@ -19,6 +19,7 @@ steps: - pytest -v -s basic_correctness/test_cpu_offload.py mirror: amd: + label: ":amd: (MI300) Basic Correctness" dind: false device: mi300_1 timeout_in_minutes: 60 diff --git a/.buildkite/test_areas/benchmarks.yaml b/.buildkite/test_areas/benchmarks.yaml index ccf3aa9cb8..4010b8d0b8 100644 --- a/.buildkite/test_areas/benchmarks.yaml +++ b/.buildkite/test_areas/benchmarks.yaml @@ -2,7 +2,7 @@ group: Benchmarks depends_on: - image-build steps: -- label: Benchmarks CLI Test +- label: ":nvidia: (H200) Benchmarks CLI" key: benchmarks-cli-test timeout_in_minutes: 45 device: h200_18gb @@ -14,13 +14,14 @@ steps: - pytest -v -s benchmarks/ mirror: amd: + label: ":amd: (MI300) Benchmarks CLI" dind: false device: mi300_1 timeout_in_minutes: 40 depends_on: - image-build-amd -- label: Attention Benchmarks Smoke Test (B200) +- label: ":nvidia: (B200) Attention Benchmark Smoke" key: attention-benchmarks-smoke-test-b200 device: b200-k8s num_gpus: 2 diff --git a/.buildkite/test_areas/compile.yaml b/.buildkite/test_areas/compile.yaml index 3ce4d7709e..dc847bf2c3 100644 --- a/.buildkite/test_areas/compile.yaml +++ b/.buildkite/test_areas/compile.yaml @@ -2,7 +2,7 @@ group: Compile depends_on: - image-build steps: -- label: Sequence Parallel Correctness Tests (2xB200) +- label: ":nvidia: (B200) Sequence Parallel Correctness" key: sequence-parallel-correctness-tests-2xb200 timeout_in_minutes: 45 working_dir: "/vllm-workspace/" @@ -13,7 +13,7 @@ steps: - export VLLM_TEST_CLEAN_GPU_MEMORY=1 - pytest -v -s tests/compile/correctness_e2e/test_sequence_parallel.py -- label: AsyncTP Correctness Tests (2xB200) +- label: ":nvidia: (B200) AsyncTP Correctness" key: asynctp-correctness-tests-b200 timeout_in_minutes: 30 working_dir: "/vllm-workspace/" @@ -24,7 +24,7 @@ steps: - export VLLM_TEST_CLEAN_GPU_MEMORY=1 - pytest -v -s tests/compile/correctness_e2e/test_async_tp.py -- label: Distributed Compile Unit Tests (2xH100) +- label: ":nvidia: (H100) Distributed Compile" key: distributed-compile-unit-tests-2xh100 timeout_in_minutes: 45 working_dir: "/vllm-workspace/" @@ -38,7 +38,7 @@ steps: - export VLLM_TEST_CLEAN_GPU_MEMORY=1 - pytest -s -v tests/compile/passes/distributed -- label: Fusion and Compile Unit Tests (2xB200) +- label: ":nvidia: (B200) Fusion and Compile" key: fusion-and-compile-unit-tests-2xb200 timeout_in_minutes: 30 working_dir: "/vllm-workspace/" @@ -65,7 +65,7 @@ steps: # this runner has 2 GPUs available even though num_devices=2 is not set - pytest -v -s tests/compile/passes/distributed/test_fusion_all_reduce.py -- label: Fusion E2E Quick (H100) +- label: ":nvidia: (H100) Fusion E2E Quick" key: fusion-e2e-quick-h100 timeout_in_minutes: 25 working_dir: "/vllm-workspace/" @@ -84,7 +84,7 @@ steps: # Qwen/Deepseek requires +quant_fp8 as -quant_fp8 rms+quant fusion is not supported - pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "inductor_partition and not +rms_norm and +quant_fp8 and (qwen3 or deepseek)" -- label: Fusion E2E Config Sweep (H100) +- label: ":nvidia: (H100) Fusion E2E Config Sweep" key: fusion-e2e-config-sweep-h100 timeout_in_minutes: 25 working_dir: "/vllm-workspace/" @@ -104,7 +104,7 @@ steps: # Run just llama3 (fp8) for all config combinations - pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "llama-3" -- label: Fusion E2E Config Sweep (B200) +- label: ":nvidia: (B200) Fusion E2E Config Sweep" key: fusion-e2e-config-sweep-b200 timeout_in_minutes: 30 working_dir: "/vllm-workspace/" @@ -118,7 +118,7 @@ steps: # Run just llama3 (fp8 & fp4) for all config combinations (only inductor partition) - pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "inductor_partition and (FLASHINFER and not +rms_norm and (not +quant_fp8 or +quant_fp8 and (qwen3 or deepseek)) or llama-3)" -- label: Fusion E2E TP2 Quick (H100) +- label: ":nvidia: (H100) Fusion E2E TP2 Quick" key: fusion-e2e-tp2-quick-h100 timeout_in_minutes: 35 working_dir: "/vllm-workspace/" @@ -136,7 +136,7 @@ steps: - pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py -k "inductor_partition and not +rms_norm and (not +quant_fp8 or +quant_fp8 and (qwen3 or deepseek))" - pytest -v -s tests/compile/fusions_e2e/test_tp2_async_tp.py -k "inductor_partition and not +rms_norm and (not +quant_fp8 or +quant_fp8 and (qwen3 or deepseek))" -- label: Fusion E2E TP2 AR-RMS Config Sweep (H100) +- label: ":nvidia: (H100) Fusion E2E TP2 AR-RMS Config Sweep" key: fusion-e2e-tp2-ar-rms-config-sweep-h100 timeout_in_minutes: 30 working_dir: "/vllm-workspace/" @@ -156,7 +156,7 @@ steps: # Run just llama3 (fp8 & bf16) for all config combinations - pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py -k "llama-3" -- label: Fusion E2E TP2 AsyncTP Config Sweep (H100) +- label: ":nvidia: (H100) Fusion E2E TP2 AsyncTP Config Sweep" key: fusion-e2e-tp2-asynctp-config-sweep-h100 timeout_in_minutes: 40 working_dir: "/vllm-workspace/" @@ -176,7 +176,7 @@ steps: # Run just llama3 (fp8 & bf16) for all config combinations - pytest -v -s tests/compile/fusions_e2e/test_tp2_async_tp.py -k "llama-3" -- label: Fusion E2E TP2 (B200) +- label: ":nvidia: (B200) Fusion E2E TP2" key: fusion-e2e-tp2-b200 timeout_in_minutes: 45 working_dir: "/vllm-workspace/" diff --git a/.buildkite/test_areas/cuda.yaml b/.buildkite/test_areas/cuda.yaml index 431ce07af4..7ddbaeec65 100644 --- a/.buildkite/test_areas/cuda.yaml +++ b/.buildkite/test_areas/cuda.yaml @@ -2,7 +2,7 @@ group: CUDA depends_on: - image-build steps: -- label: Platform Tests +- label: ":nvidia: (H200) CUDA Platform" key: platform-tests timeout_in_minutes: 20 device: h200_18gb @@ -18,7 +18,7 @@ steps: - pytest -v -s cuda/test_platform_no_cuda_init.py - pytest -v -s cuda/test_cuda_compatibility_path.py -- label: Cudagraph +- label: ":nvidia: (H200) CUDAGraph" device: h200_35gb key: cudagraph timeout_in_minutes: 30 diff --git a/.buildkite/test_areas/disaggregated.yaml b/.buildkite/test_areas/disaggregated.yaml index dd662168d2..dec438a7be 100644 --- a/.buildkite/test_areas/disaggregated.yaml +++ b/.buildkite/test_areas/disaggregated.yaml @@ -2,7 +2,7 @@ group: Disaggregated depends_on: - image-build steps: -- label: Distributed NixlConnector PD accuracy (4 GPUs) +- label: ":nvidia: (L4) Distributed NixlConnector PD accuracy" key: distributed-nixlconnector-pd-accuracy-4-gpus timeout_in_minutes: 55 working_dir: "/vllm-workspace/tests" @@ -15,6 +15,7 @@ steps: - bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh mirror: amd: + label: ":amd: (MI300) Distributed NixlConnector PD accuracy" dind: false device: mi300_4 timeout_in_minutes: 60 @@ -28,7 +29,7 @@ steps: - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh -- label: Distributed FlashInfer NixlConnector PD accuracy (4 GPUs) +- label: ":nvidia: (L4) Distributed FlashInfer NixlConnector PD accuracy" key: distributed-flashinfer-nixlconnector-pd-accuracy-4-gpus timeout_in_minutes: 55 working_dir: "/vllm-workspace/tests" @@ -40,7 +41,7 @@ steps: - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh - FLASHINFER=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh -- label: Push NixlConnector PP prefill PD accuracy (4 GPUs) +- label: ":nvidia: (L4) Push NixlConnector PP prefill PD accuracy" key: push-nixlconnector-pp-prefill-pd-accuracy-4-gpus timeout_in_minutes: 30 working_dir: "/vllm-workspace/tests" @@ -53,7 +54,7 @@ steps: - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh - bash v1/kv_connector/nixl_push_integration/config_sweep_accuracy_test.sh -- label: DP EP Distributed NixlConnector PD accuracy tests (4 GPUs) +- label: ":nvidia: (L4) DP EP Distributed NixlConnector PD accuracy" key: dp-ep-distributed-nixlconnector-pd-accuracy-tests-4-gpus timeout_in_minutes: 30 working_dir: "/vllm-workspace/tests" @@ -66,6 +67,7 @@ steps: - DP_EP=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh mirror: amd: + label: ":amd: (MI300) DP EP Distributed NixlConnector PD accuracy" dind: false device: mi300_4 timeout_in_minutes: 40 @@ -79,7 +81,7 @@ steps: - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt - DP_EP=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh -- label: CrossLayer KV layout Distributed NixlConnector PD accuracy tests (4 GPUs) +- label: ":nvidia: (L4) CrossLayer KV layout Distributed NixlConnector PD accuracy" key: crosslayer-kv-layout-distributed-nixlconnector-pd-accuracy-tests-4-gpus timeout_in_minutes: 55 working_dir: "/vllm-workspace/tests" @@ -92,6 +94,7 @@ steps: - CROSS_LAYERS_BLOCKS=True bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh mirror: amd: + label: ":amd: (MI300) CrossLayer KV layout Distributed NixlConnector PD accuracy" dind: false device: mi300_4 timeout_in_minutes: 60 @@ -105,7 +108,7 @@ steps: - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt - CROSS_LAYERS_BLOCKS=True ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh -- label: Hybrid SSM NixlConnector PD accuracy tests (4 GPUs) +- label: ":nvidia: (L4) Hybrid SSM NixlConnector PD accuracy" key: hybrid-ssm-nixlconnector-pd-accuracy-tests-4-gpus timeout_in_minutes: 60 working_dir: "/vllm-workspace/tests" @@ -118,6 +121,7 @@ steps: - HYBRID_SSM=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh mirror: amd: + label: ":amd: (MI300) Hybrid SSM NixlConnector PD accuracy" dind: false device: mi300_4 timeout_in_minutes: 55 @@ -131,7 +135,7 @@ steps: - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt - HYBRID_SSM=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh -- label: NixlConnector PD edge case test (2 GPUs) +- label: ":nvidia: (L4) NixlConnector PD edge case" key: nixlconnector-pd-edge-cases-2-gpus timeout_in_minutes: 40 working_dir: "/vllm-workspace/tests" @@ -147,7 +151,7 @@ steps: - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh - bash v1/kv_connector/nixl_integration/run_edge_case_test.sh -- label: Hybrid SSM NixlConnector PD prefix cache test (2 GPUs) +- label: ":nvidia: (L4) Hybrid SSM NixlConnector PD prefix cache" key: hybrid-ssm-nixlconnector-pd-prefix-cache-2-gpus timeout_in_minutes: 25 working_dir: "/vllm-workspace/tests" @@ -161,7 +165,7 @@ steps: - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh - bash v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh -- label: MultiConnector (Nixl+Offloading) PD accuracy (2 GPUs) +- label: ":nvidia: (L4) MultiConnector (Nixl+Offloading) PD accuracy" key: multiconnector-nixl-offloading-pd-accuracy-2-gpus timeout_in_minutes: 40 working_dir: "/vllm-workspace/tests" @@ -176,7 +180,7 @@ steps: - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh - bash v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh -- label: NixlConnector PD + Spec Decode acceptance (2 GPUs) +- label: ":nvidia: (A100) NixlConnector PD + Spec Decode acceptance" key: nixlconnector-pd-spec-decode-acceptance-2-gpus timeout_in_minutes: 45 device: a100 @@ -191,6 +195,7 @@ steps: - bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh mirror: amd: + label: ":amd: (MI300) NixlConnector PD + Spec Decode acceptance" dind: false device: mi300_2 timeout_in_minutes: 45 @@ -205,7 +210,7 @@ steps: - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt - KV_CACHE_MEMORY_BYTES=8G ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh -- label: MultiConnector (Nixl+Offloading) PD edge cases (2 GPUs) +- label: ":nvidia: (L4) MultiConnector (Nixl+Offloading) PD edge cases" key: multiconnector-nixl-offloading-pd-edge-cases-2-gpus timeout_in_minutes: 25 working_dir: "/vllm-workspace/tests" @@ -221,7 +226,7 @@ steps: - bash v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh # P TP 4 - D DPEP 4 test case for DSv4-Flash -- label: DSv4-Flash Disaggregated DP EP +- label: ":nvidia: (H200) DSv4-Flash Disaggregated DP EP" key: dsv4-flash-disaggregated timeout_in_minutes: 60 device: h200 @@ -242,7 +247,7 @@ steps: - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh - bash v1/kv_connector/nixl_integration/run_accuracy_test.sh -- label: Kimi-Linear-48B-A3B Disaggregated DP EP +- label: ":nvidia: (H200) Kimi-Linear-48B-A3B Disaggregated DP EP" key: kimi-linear-disaggregated timeout_in_minutes: 60 device: h200 diff --git a/.buildkite/test_areas/disaggregated_mooncake.yaml b/.buildkite/test_areas/disaggregated_mooncake.yaml index 92a9e625d1..0911507eb2 100644 --- a/.buildkite/test_areas/disaggregated_mooncake.yaml +++ b/.buildkite/test_areas/disaggregated_mooncake.yaml @@ -2,7 +2,7 @@ group: Disaggregated Mooncake depends_on: - image-build steps: -- label: Distributed MooncakeConnector PD accuracy (4 GPUs) +- label: ":nvidia: (B200) Distributed MooncakeConnector PD accuracy" key: distributed-mooncakeconnector-pd-accuracy-4-gpus timeout_in_minutes: 10 working_dir: "/vllm-workspace/tests" diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml index 32d1ff0342..b4b2b5ddc6 100644 --- a/.buildkite/test_areas/distributed.yaml +++ b/.buildkite/test_areas/distributed.yaml @@ -2,7 +2,7 @@ group: Distributed depends_on: - image-build steps: -- label: Distributed Comm Ops +- label: ":nvidia: (L4) Distributed Comm Ops" key: distributed-comm-ops timeout_in_minutes: 25 working_dir: "/vllm-workspace/tests" @@ -17,7 +17,7 @@ steps: - pytest -v -s distributed/test_shm_buffer.py - pytest -v -s distributed/test_shm_storage.py -- label: Distributed DP Tests (2 GPUs) +- label: ":nvidia: (L4) Distributed DP Basic" key: distributed-dp-tests-2-gpus timeout_in_minutes: 35 working_dir: "/vllm-workspace/tests" @@ -41,6 +41,7 @@ steps: - DP_SIZE=2 pytest -v -s entrypoints/openai/test_multi_api_servers.py mirror: amd: + label: ":amd: (MI300) Distributed DP Basic" dind: false device: mi300_2 timeout_in_minutes: 45 @@ -58,7 +59,7 @@ steps: - tests/entrypoints/openai/test_multi_api_servers.py - vllm/platforms/rocm.py -- label: Distributed Compile + RPC Tests (2 GPUs) +- label: ":nvidia: (L4) Distributed Compile + RPC" key: distributed-compile-rpc-tests-2-gpus timeout_in_minutes: 65 working_dir: "/vllm-workspace/tests" @@ -80,7 +81,7 @@ steps: - pytest -v -s entrypoints/llm/test_collective_rpc.py - pytest -v -s ./compile/test_wrapper.py -- label: Distributed Torchrun + Shutdown Tests (2 GPUs) +- label: ":nvidia: (L4) Distributed Torchrun + Shutdown" key: distributed-torchrun-shutdown-tests-2-gpus timeout_in_minutes: 30 working_dir: "/vllm-workspace/tests" @@ -104,7 +105,7 @@ steps: - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s v1/shutdown - pytest -v -s v1/worker/test_worker_memory_snapshot.py -- label: Distributed Torchrun + Examples (4 GPUs) +- label: ":nvidia: (L4) Distributed Torchrun + Examples" key: distributed-torchrun-examples-4-gpus timeout_in_minutes: 30 working_dir: "/vllm-workspace" @@ -137,7 +138,7 @@ steps: - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_nccl.py - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_ipc.py -- label: Distributed DP Tests (4 GPUs) +- label: ":nvidia: (L4) Distributed DP Extended" key: distributed-dp-tests-4-gpus timeout_in_minutes: 45 working_dir: "/vllm-workspace/tests" @@ -160,7 +161,7 @@ steps: - pytest -v -s v1/engine/test_engine_core_client.py::test_kv_cache_events_dp - pytest -v -s distributed/test_utils.py -- label: Distributed Compile + Comm (4 GPUs) +- label: ":nvidia: (L4) Distributed Compile + Comm" key: distributed-compile-comm-4-gpus timeout_in_minutes: 70 working_dir: "/vllm-workspace/tests" @@ -181,7 +182,7 @@ steps: # test multi-node TP with multiproc executor (simulated on single node) - pytest -v -s distributed/test_multiproc_executor.py::test_multiproc_executor_multi_node -- label: Distributed Tests (8xH100) +- label: ":nvidia: (H100) Distributed DP + EP" key: distributed-tests-8xh100 timeout_in_minutes: 20 device: h100 @@ -203,7 +204,7 @@ steps: # test with torchrun tp=2 and dp=4 with ep - torchrun --nproc-per-node=8 ../examples/features/torchrun/torchrun_dp_example_offline.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep -- label: Distributed Tests (4xA100) +- label: ":nvidia: (A100) Distributed" key: distributed-tests-4xa100 device: a100 optional: true @@ -219,7 +220,7 @@ steps: - TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' - pytest -v -s -x lora/test_mixtral.py -- label: Distributed Tests (2xH100-2xMI300) +- label: ":nvidia: (H100) Distributed Features" key: distributed-tests-2xh100-2xmi300 timeout_in_minutes: 30 device: h100 @@ -234,7 +235,7 @@ steps: - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py - pytest -v -s tests/distributed/test_packed_tensor.py -- label: Distributed Tests (2xB200) +- label: ":nvidia: (B200) Distributed" key: distributed-tests-2xb200 device: b200-k8s optional: true @@ -249,7 +250,7 @@ steps: -- label: 2 Node Test (4 GPUs) +- label: ":nvidia: (L4) Distributed 2-Node" key: 2-node-test-4-gpus timeout_in_minutes: 30 working_dir: "/vllm-workspace/tests" @@ -268,7 +269,7 @@ steps: commands: - ./.buildkite/scripts/run-multi-node-test.sh /vllm-workspace/tests 2 2 $IMAGE_TAG "VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed' && NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed' && python3 ../examples/features/data_parallel/data_parallel_offline.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=0 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code && VLLM_MULTI_NODE=1 pytest -v -s distributed/test_multi_node_assignment.py && VLLM_MULTI_NODE=1 pytest -v -s distributed/test_pipeline_parallel.py" "VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed' && NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed' && python3 ../examples/features/data_parallel/data_parallel_offline.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=1 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code" -- label: Pipeline + Context Parallelism (4 GPUs) +- label: ":nvidia: (L4) Pipeline + Context Parallelism" key: pipeline-context-parallelism-4-gpus timeout_in_minutes: 55 working_dir: "/vllm-workspace/tests" @@ -284,7 +285,7 @@ steps: - pytest -v -s distributed/test_pp_cudagraph.py - pytest -v -s distributed/test_pipeline_parallel.py -- label: RayExecutorV2 (4 GPUs) +- label: ":nvidia: (L4) RayExecutorV2" key: rayexecutorv2-4-gpus timeout_in_minutes: 45 working_dir: "/vllm-workspace/tests" diff --git a/.buildkite/test_areas/docker.yaml b/.buildkite/test_areas/docker.yaml index 9f0562ca3b..4bd36fad18 100644 --- a/.buildkite/test_areas/docker.yaml +++ b/.buildkite/test_areas/docker.yaml @@ -2,7 +2,7 @@ group: Docker depends_on: - image-build-cpu steps: -- label: Docker Build Metadata +- label: ":computer: (CPU) Docker Build Metadata" timeout_in_minutes: 20 device: cpu-small source_file_dependencies: diff --git a/.buildkite/test_areas/e2e_integration.yaml b/.buildkite/test_areas/e2e_integration.yaml index 6655ae781e..8f0e2042bf 100644 --- a/.buildkite/test_areas/e2e_integration.yaml +++ b/.buildkite/test_areas/e2e_integration.yaml @@ -2,7 +2,7 @@ group: E2E Integration depends_on: - image-build steps: -- label: DeepSeek V2-Lite Sync EPLB Accuracy (4xH100) +- label: ":nvidia: (H100) DeepSeek V2-Lite Sync EPLB Accuracy" key: deepseek-v2-lite-sync-eplb-accuracy-4xh100 timeout_in_minutes: 25 device: h100 @@ -12,7 +12,7 @@ steps: commands: - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010 -- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (4xH100) +- label: ":nvidia: (H100) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy" key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-4xh100 timeout_in_minutes: 25 device: h100 @@ -22,7 +22,7 @@ steps: commands: - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 -- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (2xB200) +- label: ":nvidia: (B200) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy" key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-2xb200 timeout_in_minutes: 20 device: b200-k8s @@ -32,7 +32,7 @@ steps: commands: - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 2 1 -- label: Qwen3-30B-A3B-FP8 DP4 Async EPLB Accuracy +- label: ":nvidia: (H100) Qwen3-30B-A3B-FP8 DP4 Async EPLB Accuracy" key: qwen3-30b-a3b-fp8-dp4-async-eplb-accuracy timeout_in_minutes: 25 device: h100 @@ -42,7 +42,7 @@ steps: commands: - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_dp4_async_eplb.sh 0.8 200 8050 -- label: DeepSeek V2-Lite Prefetch Offload Accuracy (H100) +- label: ":nvidia: (H100) DeepSeek V2-Lite Prefetch Offload Accuracy" key: deepseek-v2-lite-prefetch-offload-accuracy-h100 timeout_in_minutes: 20 device: h100 diff --git a/.buildkite/test_areas/engine.yaml b/.buildkite/test_areas/engine.yaml index 4cb661ac16..d7bb76c69e 100644 --- a/.buildkite/test_areas/engine.yaml +++ b/.buildkite/test_areas/engine.yaml @@ -2,7 +2,7 @@ group: Engine depends_on: - image-build steps: -- label: Engine +- label: ":nvidia: (H200) Engine Core" key: engine timeout_in_minutes: 30 device: h200_18gb @@ -29,13 +29,14 @@ steps: - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py jit_monitor/test_hooks.py jit_monitor/test_hooks_gpu.py mirror: amd: + label: ":amd: (MI300) Engine Core" dind: false device: mi300_1 timeout_in_minutes: 40 depends_on: - image-build-amd -- label: Engine (1 GPU) +- label: ":nvidia: (L4) V1 Engine" key: engine-1-gpu timeout_in_minutes: 45 source_file_dependencies: @@ -48,12 +49,13 @@ steps: - pytest -v -s v1/test_tensor_ipc_queue.py mirror: amd: + label: ":amd: (MI250) V1 Engine" device: mi250_1 timeout_in_minutes: 45 depends_on: - image-build-amd -- label: e2e Scheduling (1 GPU) +- label: ":nvidia: (H200) E2E Scheduling" key: e2e-scheduling-1-gpu timeout_in_minutes: 53 device: h200_18gb @@ -64,12 +66,13 @@ steps: - pytest -v -s v1/e2e/general/test_async_scheduling.py mirror: amd: + label: ":amd: (MI250) E2E Scheduling" device: mi250_1 timeout_in_minutes: 55 depends_on: - image-build-amd -- label: e2e Core (1 GPU) +- label: ":nvidia: (H200) E2E Core" device: h200_35gb key: e2e-core-1-gpu timeout_in_minutes: 40 @@ -80,6 +83,7 @@ steps: - pytest -v -s v1/e2e/general --ignore v1/e2e/general/test_async_scheduling.py mirror: amd: + label: ":amd: (MI250) E2E Core" device: mi250_1 timeout_in_minutes: 50 depends_on: @@ -89,7 +93,7 @@ steps: - tests/v1/e2e/general/ - vllm/platforms/rocm.py -- label: V1 e2e (2 GPUs) +- label: ":nvidia: (L4) V1 E2E" key: v1-e2e-2-gpus timeout_in_minutes: 25 # TODO: Fix timeout after we have more confidence in the test stability optional: true @@ -122,13 +126,14 @@ steps: v1/e2e/spec_decode/draft_model/test_draft_model.py::test_draft_model_engine_args_tensor_parallelism mirror: amd: + label: ":amd: (MI300) V1 E2E" dind: false device: mi300_2 timeout_in_minutes: 30 depends_on: - image-build-amd -- label: V1 e2e (4xH100) +- label: ":nvidia: (H100) V1 E2E" key: v1-e2e-4xh100 timeout_in_minutes: 35 device: h100 diff --git a/.buildkite/test_areas/entrypoints.yaml b/.buildkite/test_areas/entrypoints.yaml index d36f98bd67..ed143b8812 100644 --- a/.buildkite/test_areas/entrypoints.yaml +++ b/.buildkite/test_areas/entrypoints.yaml @@ -2,7 +2,7 @@ group: Entrypoints depends_on: - image-build steps: -- label: Entrypoints Unit Tests +- label: ":nvidia: (H200) Entrypoints Unit" device: h200_35gb key: entrypoints-unit-tests timeout_in_minutes: 25 @@ -15,7 +15,7 @@ steps: - pytest -v -s entrypoints/unit_tests - pytest -v -s entrypoints/weight_transfer -- label: Entrypoints Integration (LLM) +- label: ":nvidia: (H200) Entrypoints Integration (LLM)" device: h200_35gb key: entrypoints-integration-llm timeout_in_minutes: 60 @@ -31,13 +31,14 @@ steps: - pytest -v -s entrypoints/llm/offline_mode # Needs to avoid interference with other tests mirror: amd: + label: ":amd: (MI300) Entrypoints Integration (LLM)" dind: false device: mi300_1 timeout_in_minutes: 55 depends_on: - image-build-amd -- label: Entrypoints Integration (API Server) +- label: ":nvidia: (H200) Entrypoints Integration (API Server)" key: entrypoints-integration-api-server device: h200_35gb timeout_in_minutes: 75 @@ -54,13 +55,14 @@ steps: - pytest -v -s entrypoints/scale_out mirror: amd: + label: ":amd: (MI300) Entrypoints Integration (API Server)" dind: false device: mi300_1 timeout_in_minutes: 65 depends_on: - image-build-amd -- label: Entrypoints Integration (API Server OpenAI - Part 1) +- label: ":nvidia: (H200) Entrypoints Integration (API Server OpenAI - Part 1)" device: h200_35gb key: entrypoints-integration-api-server-openai-part-1 timeout_in_minutes: 68 @@ -75,13 +77,14 @@ steps: - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness mirror: amd: + label: ":amd: (MI300) Entrypoints Integration (API Server OpenAI - Part 1)" dind: false device: mi300_1 timeout_in_minutes: 65 depends_on: - image-build-amd -- label: Entrypoints Integration (API Server OpenAI - Part 2) +- label: ":nvidia: (H200) Entrypoints Integration (API Server OpenAI - Part 2)" device: h200_35gb key: entrypoints-integration-api-server-openai-part-2 timeout_in_minutes: 83 @@ -97,13 +100,14 @@ steps: - pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py mirror: amd: + label: ":amd: (MI300) Entrypoints Integration (API Server OpenAI - Part 2)" dind: false device: mi300_1 timeout_in_minutes: 70 depends_on: - image-build-amd -- label: Entrypoints Integration (API Server Generate) +- label: ":nvidia: (H200) Entrypoints Integration (API Server Generate)" device: h200_35gb key: entrypoints-integration-api-server-generate timeout_in_minutes: 50 @@ -122,13 +126,14 @@ steps: - pytest -v -s entrypoints/anthropic mirror: amd: + label: ":amd: (MI300) Entrypoints Integration (API Server Generate)" dind: false device: mi300_1 timeout_in_minutes: 65 depends_on: - image-build-amd -- label: Entrypoints Integration (Responses API) +- label: ":nvidia: (H200) Entrypoints Integration (Responses API)" device: h200_35gb key: entrypoints-integration-responses-api timeout_in_minutes: 50 @@ -140,7 +145,7 @@ steps: commands: - pytest -v -s entrypoints/openai/responses -- label: Entrypoints Integration (Speech to Text) +- label: ":nvidia: (H200) Entrypoints Integration (Speech to Text)" device: h200_35gb key: entrypoints-integration-speech_to_text timeout_in_minutes: 45 @@ -153,7 +158,7 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/speech_to_text -- label: Entrypoints Integration (Multimodal) +- label: ":nvidia: (H200) Entrypoints Integration (Multimodal)" device: h200_35gb key: entrypoints-integration-multimodal timeout_in_minutes: 45 @@ -166,7 +171,7 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/multimodal -- label: Entrypoints Integration (Pooling) +- label: ":nvidia: (H200) Entrypoints Integration (Pooling)" device: h200_35gb key: entrypoints-integration-pooling timeout_in_minutes: 75 @@ -179,7 +184,7 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/pooling -- label: OpenAI API Correctness +- label: ":nvidia: (H200) OpenAI API Correctness" key: openai-api-correctness timeout_in_minutes: 20 device: h200_18gb @@ -190,6 +195,7 @@ steps: - pytest -s entrypoints/openai/correctness/ mirror: amd: + label: ":amd: (MI300) OpenAI API Correctness" dind: false device: mi300_1 timeout_in_minutes: 30 diff --git a/.buildkite/test_areas/expert_parallelism.yaml b/.buildkite/test_areas/expert_parallelism.yaml index 89c9fddf64..e0b931f5fe 100644 --- a/.buildkite/test_areas/expert_parallelism.yaml +++ b/.buildkite/test_areas/expert_parallelism.yaml @@ -2,7 +2,7 @@ group: Expert Parallelism depends_on: - image-build steps: -- label: EPLB Algorithm +- label: ":nvidia: (H200) EPLB Algorithm" key: eplb-algorithm timeout_in_minutes: 20 device: h200_18gb @@ -16,6 +16,7 @@ steps: - pytest -v -s distributed/test_eplb_utils.py mirror: amd: + label: ":amd: (MI300) EPLB Algorithm" dind: false device: mi300_1 timeout_in_minutes: 30 @@ -27,7 +28,7 @@ steps: - tests/distributed/test_eplb_utils.py - vllm/platforms/rocm.py -- label: EPLB Execution # 17min +- label: ":nvidia: (L4) EPLB Execution" key: eplb-execution timeout_in_minutes: 25 working_dir: "/vllm-workspace/tests" @@ -39,7 +40,7 @@ steps: - pytest -v -s distributed/test_eplb_execute.py - pytest -v -s distributed/test_eplb_spec_decode.py -- label: Elastic EP Scaling Test +- label: ":nvidia: (H100) Elastic EP Scaling" key: elastic-ep-scaling-test timeout_in_minutes: 30 device: h100 diff --git a/.buildkite/test_areas/fault_tolerance.yaml b/.buildkite/test_areas/fault_tolerance.yaml index e2f700a8bd..2e85a1c3f8 100644 --- a/.buildkite/test_areas/fault_tolerance.yaml +++ b/.buildkite/test_areas/fault_tolerance.yaml @@ -2,7 +2,7 @@ group: Fault Tolerance depends_on: - image-build steps: -- label: Fault Tolerance E2E (2xH100) +- label: ":nvidia: (H100) Fault Tolerance E2E" key: fault-tolerance-e2e-2xh100 timeout_in_minutes: 35 device: h100 diff --git a/.buildkite/test_areas/jit_monitor.yaml b/.buildkite/test_areas/jit_monitor.yaml index 7119c489c0..941c190138 100644 --- a/.buildkite/test_areas/jit_monitor.yaml +++ b/.buildkite/test_areas/jit_monitor.yaml @@ -2,7 +2,7 @@ group: JIT Monitor depends_on: - image-build steps: -- label: No Runtime JITs e2e tests +- label: ":nvidia: (H200) No Runtime JITs E2E" key: jit-monitor-no-runtime-jit device: h200_35gb timeout_in_minutes: 45 diff --git a/.buildkite/test_areas/kernels.yaml b/.buildkite/test_areas/kernels.yaml index 4dec86e929..0887a6fb27 100644 --- a/.buildkite/test_areas/kernels.yaml +++ b/.buildkite/test_areas/kernels.yaml @@ -2,7 +2,7 @@ group: Kernels depends_on: - image-build steps: -- label: vLLM IR Tests +- label: ":nvidia: (H200) vLLM IR" key: vllm-ir-tests timeout_in_minutes: 35 device: h200_18gb @@ -14,7 +14,7 @@ steps: - pytest -v -s tests/ir - pytest -v -s tests/kernels/ir -- label: Kernels Core Operation Test +- label: ":nvidia: (H200) Core Operation Kernels Shard %N" device: h200_35gb key: kernels-core-operation-test timeout_in_minutes: 120 @@ -27,7 +27,7 @@ steps: - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_concat_mla_q.py kernels/test_fused_qk_norm_rope_gate.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT parallelism: 3 -- label: Kernels MiniMax Reduce RMS Test (2 GPUs) +- label: ":nvidia: (H100) MiniMax Reduce RMS Kernels" key: kernels-minimax-reduce-rms-test-2-gpus timeout_in_minutes: 20 num_devices: 2 @@ -41,7 +41,7 @@ steps: commands: - pytest -v -s kernels/core/test_minimax_reduce_rms.py -- label: Deepseek V4 Kernel Test (H100) +- label: ":nvidia: (H100) DeepSeek V4 Kernels" key: deepseek-v4-kernel-test-h100 timeout_in_minutes: 30 device: h100 @@ -54,7 +54,7 @@ steps: - pytest -v -s kernels/test_fused_deepseek_v4_*.py - pytest -v -s kernels/test_top_k_per_row.py -- label: Deepseek V4 Kernel Test (B200) +- label: ":nvidia: (B200) DeepSeek V4 Kernels" key: deepseek-v4-kernel-test-b200 timeout_in_minutes: 20 device: b200-k8s @@ -73,7 +73,7 @@ steps: # Files with dedicated jobs elsewhere in this file are excluded via --ignore # (test_kda, test_bf16x3_router_gemm_cutedsl and test_ll_bf16_gemm run in # their own jobs / Kernels (B200)). -- label: Kernels Root Misc Test (B200) +- label: ":nvidia: (B200) Miscellaneous Kernels" key: kernels-root-misc-test-b200 timeout_in_minutes: 45 device: b200-k8s @@ -101,7 +101,7 @@ steps: # BROKEN on main, pending kernel fixes (B200): # test_shuffle_rows.py (1: test_shuffle_rows_edge_cases) -- label: Kernels Attention Test %N +- label: ":nvidia: (L4) Attention Kernels Shard %N" key: kernels-attention-test timeout_in_minutes: 65 source_file_dependencies: @@ -116,6 +116,7 @@ steps: parallelism: 2 mirror: amd: + label: ":amd: (MI300) Attention Kernels Shard %N" dind: false device: mi300_1 timeout_in_minutes: 90 @@ -130,7 +131,7 @@ steps: - vllm/envs.py - vllm/platforms/rocm.py -- label: Kernels Attention DiffKV Test (H100) +- label: ":nvidia: (H100) Attention DiffKV Kernels" key: kernels-attention-diffkv-test-h100 timeout_in_minutes: 20 device: h100 @@ -143,7 +144,7 @@ steps: commands: - pytest -v -s kernels/attention/test_triton_unified_attention_diffkv.py -- label: Kernels FlashMLA Test (H100) +- label: ":nvidia: (H100) FlashMLA Kernels" key: kernels-flashmla-test-h100 timeout_in_minutes: 25 device: h100 @@ -161,7 +162,7 @@ steps: - pytest -v -s kernels/attention/test_flashmla_sparse.py - pytest -v -s kernels/attention/test_mla_cross_layer_kernel_equivalence.py -- label: Kernels Quantization Test %N +- label: ":nvidia: (L4) Quantization Kernels Shard %N" key: kernels-quantization-test timeout_in_minutes: 60 source_file_dependencies: @@ -173,6 +174,7 @@ steps: parallelism: 2 mirror: amd: + label: ":amd: (MI300) Quantization Kernels Shard %N" dind: false device: mi300_1 timeout_in_minutes: 120 @@ -189,7 +191,7 @@ steps: depends_on: - image-build-amd -- label: Kernels MoE Test %N +- label: ":nvidia: (L4) MoE Kernels Shard %N" key: kernels-moe-test timeout_in_minutes: 50 source_file_dependencies: @@ -206,6 +208,7 @@ steps: parallelism: 5 mirror: amd: + label: ":amd: (MI300) MoE Kernels Shard %N" dind: false device: mi300_1 timeout_in_minutes: 55 @@ -222,7 +225,7 @@ steps: depends_on: - image-build-amd -- label: Kernels Mamba Test +- label: ":nvidia: (H200) Mamba Kernels" device: h200_35gb key: kernels-mamba-test timeout_in_minutes: 60 @@ -233,7 +236,7 @@ steps: commands: - pytest -v -s kernels/mamba -- label: Kernels DeepGEMM Test (H100) +- label: ":nvidia: (H100) DeepGEMM Kernels" key: kernels-deepgemm-test-h100 timeout_in_minutes: 35 device: h100 @@ -260,7 +263,7 @@ steps: - pytest -v -s kernels/attention/test_deepgemm_attention.py - pytest -v -s quantization/test_cutlass_w4a16.py -- label: Kernels (B200) +- label: ":nvidia: (B200) Kernels" key: kernels-b200 timeout_in_minutes: 80 working_dir: "/vllm-workspace/" @@ -340,7 +343,7 @@ steps: # e2e - pytest -v -s tests/models/quantization/test_nvfp4.py -- label: B12X Linear Kernels (DGX Spark) Nightly +- label: ":nvidia: (DGX) Spark B12X Linear Kernels Nightly" key: b12x-linear-kernels-dgx-spark-nightly timeout_in_minutes: 30 device: dgx-spark @@ -362,7 +365,7 @@ steps: model_executor/test_b12x_warmup.py kernels/quantization/test_block_fp8.py -k b12x -- label: Kernels Helion Test +- label: ":nvidia: (H100) Helion Kernels Shard %N" key: kernels-helion-test timeout_in_minutes: 115 device: h100 @@ -375,7 +378,7 @@ steps: parallelism: 2 -- label: Kernels FP8 MoE Test (1xH100) +- label: ":nvidia: (H100) FP8 MoE Kernels" key: kernels-fp8-moe-test-1xh100 timeout_in_minutes: 40 device: h100 @@ -392,7 +395,7 @@ steps: - pytest -v -s kernels/moe/test_triton_moe_no_act_mul.py - pytest -v -s kernels/moe/test_triton_moe_ptpc_fp8.py -- label: Kernels FP8 MoE Test (2xH100) +- label: ":nvidia: (H100) DeepEP FP8 MoE Kernels" key: kernels-fp8-moe-test-2xh100 timeout_in_minutes: 45 device: h100 @@ -402,7 +405,7 @@ steps: - pytest -v -s kernels/moe/test_deepep_deepgemm_moe.py - pytest -v -s kernels/moe/test_deepep_moe.py -- label: Kernels Fp4 MoE Test (B200) +- label: ":nvidia: (B200) FP4 MoE Kernels" key: kernels-fp4-moe-test-b200 timeout_in_minutes: 25 device: b200-k8s @@ -415,7 +418,7 @@ steps: - pytest -v -s kernels/moe/test_ocp_mx_moe.py -- label: Kernels FusedMoE Layer Test (2 H100s) +- label: ":nvidia: (H100) FusedMoE Layer Kernels" key: kernels-fusedmoe-layer-test-2-h100s timeout_in_minutes: 30 device: h100 @@ -432,7 +435,7 @@ steps: - pytest -v -s kernels/moe/test_moe_layer.py -- label: Kernels FusedMoE Layer Test (2 B200s) +- label: ":nvidia: (B200) FusedMoE Layer Kernels" key: kernels-fusedmoe-layer-test-2-b200s timeout_in_minutes: 90 device: b200-k8s diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index f9c21a5421..58e6717aca 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -2,7 +2,7 @@ group: LM Eval depends_on: - image-build steps: -- label: LM Eval Small Models +- label: ":nvidia: (H200) LM Eval Small Models" device: h200_35gb key: lm-eval-small-models timeout_in_minutes: 45 @@ -14,6 +14,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt mirror: amd: + label: ":amd: (MI300) LM Eval Small Models" dind: false device: mi300_1 timeout_in_minutes: 45 @@ -42,7 +43,7 @@ steps: # - export VLLM_WORKER_MULTIPROC_METHOD=spawn # - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4 -- label: LM Eval Large Models (4xH100) +- label: ":nvidia: (H100) LM Eval Large Models" key: lm-eval-large-models-4xh100 device: h100 optional: true @@ -55,7 +56,7 @@ steps: - export VLLM_USE_DEEP_GEMM=0 # We found Triton is faster than DeepGEMM for H100 - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-hopper.txt --tp-size=4 -- label: LM Eval Small Models (1xB200) +- label: ":nvidia: (B200) LM Eval Small Models" key: lm-eval-small-models-1xb200 timeout_in_minutes: 50 device: b200-k8s @@ -66,7 +67,7 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt -- label: LM Eval Small Models Distributed (2xB200) +- label: ":nvidia: (B200) LM Eval Small Models Distributed" key: lm-eval-small-models-distributed-2xb200 timeout_in_minutes: 120 device: b200-k8s @@ -79,7 +80,7 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small-tp.txt -- label: LM Eval PCP (4xB200) +- label: ":nvidia: (B200) LM Eval PCP" key: lm-eval-pcp-4xb200 timeout_in_minutes: 360 device: b200-k8s @@ -101,7 +102,7 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-pcp.txt -- label: LM Eval Spec Decode (4xB200) +- label: ":nvidia: (B200) LM Eval Spec Decode" key: lm-eval-spec-decode-4xb200 timeout_in_minutes: 120 device: b200-k8s @@ -121,7 +122,7 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-spec-decode.txt -- label: LM Eval Large Models EP (2xB200) +- label: ":nvidia: (B200) LM Eval Large Models EP" key: lm-eval-large-models-ep-2xb200 timeout_in_minutes: 60 device: b200-k8s @@ -133,7 +134,7 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell-ep.txt -- label: LM Eval Qwen3.5 Models (2xB200) +- label: ":nvidia: (B200) LM Eval Qwen3.5 Models" key: lm-eval-qwen3-5-models-2xb200 timeout_in_minutes: 45 device: b200-k8s @@ -150,7 +151,7 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-blackwell.txt -- label: LM Eval Large Models (8xH200) +- label: ":nvidia: (H200) LM Eval Large Models" key: lm-eval-large-models-8xh200 timeout_in_minutes: 50 device: h200 @@ -160,6 +161,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200.txt mirror: amd: + label: ":amd: (MI300) LM Eval Large Models" dind: false device: mi300_8 timeout_in_minutes: 40 @@ -170,7 +172,7 @@ steps: - export PYTORCH_ROCM_ARCH=gfx942 # Limit Quark compilation to save time - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx.txt -- label: MoE Refactor Integration Test (H100 - TEMPORARY) +- label: ":nvidia: (H100) MoE Refactor Integration TEMPORARY" key: moe-refactor-integration-test-h100-temporary device: h100 optional: true @@ -178,7 +180,7 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor/config-h100.txt -- label: MoE Refactor Integration Test (B200 - TEMPORARY) %N +- label: ":nvidia: (B200) MoE Refactor Integration TEMPORARY Shard %N" key: moe-refactor-integration-test-b200-temporary device: b200-k8s optional: true @@ -187,7 +189,7 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor/config-b200-shard-$$BUILDKITE_PARALLEL_JOB.txt -- label: MoE Refactor Integration Test (B200 DP - TEMPORARY) +- label: ":nvidia: (B200) MoE Refactor DP Integration TEMPORARY" key: moe-refactor-integration-test-b200-dp-temporary device: b200-k8s optional: true @@ -195,7 +197,7 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt -- label: LM Eval Humming f16 (A100 - TEMPORARY) %N +- label: ":nvidia: (A100) LM Eval Humming FP16 TEMPORARY Shard %N" key: lm-eval-humming-f16-a100 timeout_in_minutes: 75 device: a100 @@ -211,7 +213,7 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-a100-shard-$$BUILDKITE_PARALLEL_JOB.txt -- label: LM Eval Humming Act int8 (A100 - TEMPORARY) +- label: ":nvidia: (A100) LM Eval Humming Activation INT8 TEMPORARY" key: lm-eval-humming-act-a100 timeout_in_minutes: 45 device: a100 @@ -226,7 +228,7 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt -- label: LM Eval Humming f16 (H100 - TEMPORARY) %N +- label: ":nvidia: (H100) LM Eval Humming FP16 TEMPORARY Shard %N" key: lm-eval-humming-f16-h100 timeout_in_minutes: 70 device: h100 @@ -242,7 +244,7 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-h100-shard-$$BUILDKITE_PARALLEL_JOB.txt -- label: LM Eval Humming Act fp8/int8 (H100 - TEMPORARY) +- label: ":nvidia: (H100) LM Eval Humming Activation FP8/INT8 TEMPORARY" key: lm-eval-humming-act-h100 timeout_in_minutes: 70 device: h100 @@ -258,7 +260,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt # - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt -- label: LM Eval Humming f16 (B200 - TEMPORARY) +- label: ":nvidia: (B200) LM Eval Humming FP16 TEMPORARY" key: lm-eval-humming-f16-b200 timeout_in_minutes: 50 device: b200-k8s @@ -273,7 +275,7 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt -- label: LM Eval Humming Act fp8/int8 (B200 - TEMPORARY) +- label: ":nvidia: (B200) LM Eval Humming Activation FP8/INT8 TEMPORARY" key: lm-eval-humming-act-b200 timeout_in_minutes: 50 device: b200-k8s @@ -289,7 +291,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt -- label: LM Eval TurboQuant KV Cache +- label: ":nvidia: (H200) LM Eval TurboQuant KV Cache" key: lm-eval-turboquant-kv-cache timeout_in_minutes: 55 device: h200_18gb @@ -301,7 +303,7 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant.txt -- label: GPQA Eval (GPT-OSS) (2xH100) +- label: ":nvidia: (H100) GPQA Eval (GPT-OSS)" key: gpqa-eval-gpt-oss-2xh100 timeout_in_minutes: 35 device: h100 @@ -315,7 +317,7 @@ steps: - uv pip install --system 'gpt-oss[eval]==0.0.5' - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-h100.txt -- label: GPQA Eval (GPT-OSS) (2xB200) +- label: ":nvidia: (B200) GPQA Eval (GPT-OSS)" key: gpqa-eval-gpt-oss-2xb200 timeout_in_minutes: 30 device: b200-k8s @@ -329,7 +331,7 @@ steps: - uv pip install --system 'gpt-oss[eval]==0.0.5' - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-b200.txt -- label: GPQA Eval (GPT-OSS) (DGX Spark) +- label: ":nvidia: (DGX) Spark GPQA Eval (GPT-OSS)" key: gpqa-eval-gpt-oss-spark timeout_in_minutes: 35 device: dgx-spark @@ -345,7 +347,7 @@ steps: - uv pip install --system 'gpt-oss[eval]==0.0.5' - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-spark.txt -- label: LM Eval KV-Offload (1xH200) +- label: ":nvidia: (H200) LM Eval KV-Offload" key: kv-offload-small timeout_in_minutes: 30 device: h200_35gb @@ -358,7 +360,7 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "nemotron-h-8b or gemma-4-e4b-it" -- label: LM Eval KV-Offload (2xH100) +- label: ":nvidia: (H100) LM Eval KV-Offload Medium" key: kv-offload-medium timeout_in_minutes: 45 device: h100 @@ -372,7 +374,7 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "qwen3.5-35b or deepseek-v2-lite" -- label: LM Eval KV-Offload (4xH100) +- label: ":nvidia: (H100) LM Eval KV-Offload Large" key: kv-offload-large timeout_in_minutes: 40 device: h100 @@ -386,7 +388,7 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "deepseek-v4-flash" -- label: MRCR Eval Small Models +- label: ":nvidia: (H200) MRCR Eval Small Models" device: h200_35gb timeout_in_minutes: 25 source_file_dependencies: diff --git a/.buildkite/test_areas/lora.yaml b/.buildkite/test_areas/lora.yaml index 4287816b90..5c3c7c0e89 100644 --- a/.buildkite/test_areas/lora.yaml +++ b/.buildkite/test_areas/lora.yaml @@ -2,7 +2,7 @@ group: LoRA depends_on: - image-build steps: -- label: LoRA %N +- label: ":nvidia: (H200) LoRA Shard %N" device: h200_35gb key: lora timeout_in_minutes: 40 @@ -14,6 +14,7 @@ steps: parallelism: 4 mirror: amd: + label: ":amd: (MI300) LoRA Shard %N" dind: false device: mi300_1 working_dir: "/vllm-workspace/tests" @@ -26,7 +27,7 @@ steps: - image-build-amd -- label: LoRA TP (Distributed) +- label: ":nvidia: (L4) LoRA TP (Distributed)" key: lora-tp-distributed timeout_in_minutes: 60 num_devices: 4 diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index 8f168cbd36..750f51283a 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -2,7 +2,7 @@ group: Miscellaneous depends_on: - image-build steps: -- label: V1 Sample + Logits +- label: ":nvidia: (H200) V1 Sample + Logits" key: v1-sample-logits timeout_in_minutes: 83 device: h200_18gb @@ -33,13 +33,14 @@ steps: - pytest -v -s v1/test_outputs.py mirror: amd: + label: ":amd: (MI300) V1 Sample + Logits" dind: false device: mi300_1 timeout_in_minutes: 70 depends_on: - image-build-amd -- label: V1 Core + KV + Metrics +- label: ":nvidia: (H200) V1 Core + KV + Metrics" device: h200_35gb key: v1-core-kv-metrics timeout_in_minutes: 80 @@ -90,13 +91,14 @@ steps: - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine mirror: amd: + label: ":amd: (MI300) V1 Core + KV + Metrics" dind: false device: mi300_1 timeout_in_minutes: 65 depends_on: - image-build-amd -- label: V1 Others (CPU) +- label: ":computer: (CPU) V1 Others" key: v1-others-cpu depends_on: - image-build-cpu @@ -130,7 +132,7 @@ steps: - pytest -v -s -m 'cpu_test' v1/ec_connector/unit - pytest -v -s -m 'cpu_test' v1/metrics -- label: Extract Hidden States Integration +- label: ":nvidia: (H200) Extract Hidden States Integration" key: extract-hidden-states-integration timeout_in_minutes: 20 device: h200_18gb @@ -144,7 +146,7 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s v1/kv_connector/extract_hidden_states_integration -- label: Extract Hidden States Integration (2 GPUs) +- label: ":nvidia: (L4) Extract Hidden States Integration" key: extract-hidden-states-integration-2-gpus timeout_in_minutes: 20 num_devices: 2 @@ -158,7 +160,7 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s -m 'distributed' v1/kv_connector/extract_hidden_states_integration -- label: Regression +- label: ":nvidia: (H200) Regression" key: regression timeout_in_minutes: 30 device: h200_18gb @@ -181,7 +183,7 @@ steps: - pytest -v -s test_regression.py working_dir: "/vllm-workspace/tests" # optional -- label: Examples +- label: ":nvidia: (H200) Examples" device: h200_35gb key: examples timeout_in_minutes: 40 @@ -215,6 +217,7 @@ steps: - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 mirror: amd: + label: ":amd: (MI300) Examples" dind: false device: mi300_1 timeout_in_minutes: 75 @@ -226,7 +229,7 @@ steps: depends_on: - image-build-amd -- label: Metrics, Tracing (2 GPUs) +- label: ":nvidia: (L4) Metrics, Tracing" key: metrics-tracing-2-gpus timeout_in_minutes: 25 num_devices: 2 @@ -256,13 +259,14 @@ steps: - pytest -v -s tracing mirror: amd: + label: ":amd: (MI300) Metrics, Tracing" dind: false device: mi300_2 timeout_in_minutes: 30 depends_on: - image-build-amd -- label: Python-only Installation +- label: ":nvidia: (H200) Python-only Installation" key: python-only-installation depends_on: ~ optional: true @@ -275,6 +279,7 @@ steps: - bash standalone_tests/python_only_compile.sh mirror: amd: + label: ":amd: (MI300) Python-only Installation" dind: false device: mi300_1 timeout_in_minutes: 55 @@ -286,7 +291,7 @@ steps: - setup.py - vllm/platforms/rocm.py -- label: Async Engine, Inputs, Utils, Worker +- label: ":nvidia: (H200) Async Engine, Inputs, Utils, Worker" device: h200_35gb key: async-engine-inputs-utils-worker timeout_in_minutes: 25 @@ -313,7 +318,7 @@ steps: - pytest -v -s -m 'not cpu_test' multimodal - pytest -v -s utils_ -- label: Async Engine, Inputs, Utils, Worker, Config (CPU) +- label: ":computer: (CPU) Async Engine, Inputs, Utils, Worker, Config" key: async-engine-inputs-utils-worker-config-cpu depends_on: - image-build-cpu @@ -377,7 +382,7 @@ steps: - pytest -v -s transformers_utils - pytest -v -s config -- label: Batch Invariance (A100) +- label: ":nvidia: (A100) Batch Invariance" key: batch-invariance-a100 timeout_in_minutes: 60 device: a100 @@ -391,7 +396,7 @@ steps: - pytest -v -s v1/determinism/test_batch_invariance.py - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA -- label: Batch Invariance (H100) +- label: ":nvidia: (H100) Batch Invariance" key: batch-invariance-h100 timeout_in_minutes: 60 device: h100 @@ -407,7 +412,7 @@ steps: - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA - VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k FLASH_ATTN -- label: Batch Invariance (B200) +- label: ":nvidia: (B200) Batch Invariance" key: batch-invariance-b200 timeout_in_minutes: 45 device: b200-k8s @@ -427,7 +432,7 @@ steps: - pytest -v -s v1/determinism/test_cutlass_batch_invariance.py - pytest -v -s v1/determinism/test_online_batch_invariance.py -- label: Acceptance Length Test (Large Models) # optional +- label: ":nvidia: (H200) Acceptance Length (Large Models)" device: h200_35gb key: acceptance-length-test-large-models timeout_in_minutes: 20 diff --git a/.buildkite/test_areas/model_executor.yaml b/.buildkite/test_areas/model_executor.yaml index 07d4aea871..8228bbc83a 100644 --- a/.buildkite/test_areas/model_executor.yaml +++ b/.buildkite/test_areas/model_executor.yaml @@ -2,7 +2,7 @@ group: Model Executor depends_on: - image-build steps: -- label: Model Executor +- label: ":nvidia: (H200) Model Executor" device: h200_35gb key: model-executor timeout_in_minutes: 60 @@ -28,6 +28,7 @@ steps: - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread mirror: amd: + label: ":amd: (MI300) Model Executor" dind: false device: mi300_1 timeout_in_minutes: 60 diff --git a/.buildkite/test_areas/model_runner_v2.yaml b/.buildkite/test_areas/model_runner_v2.yaml index 9b05f4b1e2..1b1bef43aa 100644 --- a/.buildkite/test_areas/model_runner_v2.yaml +++ b/.buildkite/test_areas/model_runner_v2.yaml @@ -2,7 +2,7 @@ group: Model Runner V2 depends_on: - image-build steps: -- label: Model Runner V2 Core Tests +- label: ":nvidia: (H200) Model Runner V2 Core" device: h200_35gb key: model-runner-v2-core-tests timeout_in_minutes: 35 @@ -24,7 +24,7 @@ steps: # Temporary hack filter to exclude ngram spec decoding based tests. - pytest -v -s entrypoints/llm/test_struct_output_generate.py -k "xgrammar and not speculative_config6 and not speculative_config7 and not speculative_config8 and not speculative_config0" -- label: Model Runner V2 Examples +- label: ":nvidia: (H200) Model Runner V2 Examples" device: h200_35gb key: model-runner-v2-examples timeout_in_minutes: 35 @@ -61,7 +61,7 @@ steps: # https://github.com/vllm-project/vllm/pull/26682 uses slightly more memory in PyTorch 2.9+ causing this test to OOM in 1xL4 GPU - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 -- label: Model Runner V2 Distributed (2 GPUs) +- label: ":nvidia: (L4) Model Runner V2 Distributed" key: model-runner-v2-distributed-2-gpus timeout_in_minutes: 30 working_dir: "/vllm-workspace/tests" @@ -82,7 +82,7 @@ steps: - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py -k "not ray" - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py -- label: Model Runner V2 Pipeline Parallelism (4 GPUs) +- label: ":nvidia: (L4) Model Runner V2 Pipeline Parallelism" key: model-runner-v2-pipeline-parallelism-4-gpus timeout_in_minutes: 50 working_dir: "/vllm-workspace/tests" @@ -100,7 +100,7 @@ steps: - pytest -v -s distributed/test_pp_cudagraph.py -k "not ray" - pytest -v -s v1/distributed/test_pp_dp_v2.py -- label: Model Runner V2 Spec Decode +- label: ":nvidia: (H200) Model Runner V2 Spec Decode" device: h200_35gb key: model-runner-v2-spec-decode timeout_in_minutes: 50 diff --git a/.buildkite/test_areas/models_basic.yaml b/.buildkite/test_areas/models_basic.yaml index b3d922a5e9..bdb3cab5a6 100644 --- a/.buildkite/test_areas/models_basic.yaml +++ b/.buildkite/test_areas/models_basic.yaml @@ -2,7 +2,7 @@ group: Models - Basic depends_on: - image-build steps: -- label: Basic Models Tests (Initialization) +- label: ":nvidia: (H200) Basic Models (Initialization)" key: basic-models-tests-initialization timeout_in_minutes: 25 device: h200_18gb @@ -15,7 +15,7 @@ steps: # Run a subset of model initialization tests - pytest -v -s models/test_initialization.py::test_can_initialize_small_subset -- label: Basic Models Tests (Extra Initialization) %N +- label: ":nvidia: (H200) Basic Models (Extra Initialization) Shard %N" device: h200_35gb key: basic-models-tests-extra-initialization timeout_in_minutes: 100 @@ -30,7 +30,7 @@ steps: - pytest -v -s models/test_initialization.py -k 'not test_can_initialize_small_subset' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB parallelism: 4 -- label: Basic Models Tests (Other) +- label: ":nvidia: (H200) Basic Models (Other)" device: h200_35gb key: basic-models-tests-other timeout_in_minutes: 35 @@ -44,13 +44,14 @@ steps: - pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py mirror: amd: + label: ":amd: (MI300) Basic Models (Other)" dind: false device: mi300_1 timeout_in_minutes: 50 depends_on: - image-build-amd -- label: Inkling Unit Tests (B200) +- label: ":nvidia: (B200) Inkling" key: inkling-unit-tests-b200 timeout_in_minutes: 40 device: b200-k8s @@ -63,7 +64,7 @@ steps: # FA4 kernel tests require SM100; the suite skips them elsewhere. - pytest -v -s models/inkling -- label: Kimi K3 Unit Tests (B200) +- label: ":nvidia: (B200) Kimi K3" key: kimi-k3-unit-tests-b200 timeout_in_minutes: 40 device: b200-k8s @@ -77,7 +78,7 @@ steps: # The native NVIDIA Kimi K3 kernels require the SM100 family. - pytest -v -s models/kimi_k3 kernels/attention/test_kimi_k3_mla_fused_epilogue.py kernels/test_bf16_skinny_gemm.py -- label: Basic Models Test (Other CPU) # 5min +- label: ":computer: (CPU) Basic Models Other" key: basic-models-test-other-cpu depends_on: - image-build-cpu diff --git a/.buildkite/test_areas/models_distributed.yaml b/.buildkite/test_areas/models_distributed.yaml index c1ec5eb00a..730c51b89b 100644 --- a/.buildkite/test_areas/models_distributed.yaml +++ b/.buildkite/test_areas/models_distributed.yaml @@ -2,7 +2,7 @@ group: Models - Distributed depends_on: - image-build steps: -- label: Distributed Model Tests (2 GPUs) +- label: ":nvidia: (L4) Distributed Models" key: distributed-model-tests-2-gpus timeout_in_minutes: 60 working_dir: "/vllm-workspace/tests" diff --git a/.buildkite/test_areas/models_language.yaml b/.buildkite/test_areas/models_language.yaml index d0eff968f7..e867514123 100644 --- a/.buildkite/test_areas/models_language.yaml +++ b/.buildkite/test_areas/models_language.yaml @@ -2,7 +2,7 @@ group: Models - Language depends_on: - image-build steps: -- label: Language Models Tests (Standard) +- label: ":nvidia: (H200) Language Models (Standard)" key: language-models-tests-standard timeout_in_minutes: 30 device: h200_18gb @@ -16,13 +16,14 @@ steps: - pytest -v -s models/language -m 'core_model and (not slow_test)' mirror: amd: + label: ":amd: (MI300) Language Models (Standard)" dind: false device: mi300_1 timeout_in_minutes: 45 depends_on: - image-build-amd -- label: Language Models Tests (Extra Standard) %N +- label: ":nvidia: (H200) Language Models (Extra Standard) Shard %N" device: h200_35gb key: language-models-tests-extra-standard timeout_in_minutes: 40 @@ -39,6 +40,7 @@ steps: parallelism: 2 mirror: amd: + label: ":amd: (MI300) Language Models (Extra Standard) Shard %N" dind: false device: mi300_1 timeout_in_minutes: 40 @@ -55,7 +57,7 @@ steps: - tests/models/language/pooling/test_classification.py - vllm/_aiter_ops.py - vllm/platforms/rocm.py -- label: Language Models Tests (Hybrid) %N +- label: ":nvidia: (H200) Language Models (Hybrid) Shard %N" device: h200_35gb key: language-models-tests-hybrid timeout_in_minutes: 65 @@ -72,6 +74,7 @@ steps: parallelism: 2 mirror: amd: + label: ":amd: (MI300) Language Models (Hybrid) Shard %N" dind: false device: mi300_1 timeout_in_minutes: 60 @@ -85,7 +88,7 @@ steps: # Granite 4 hybrid generation is sensitive to hardware-specific Triton SSD # autotuning (https://github.com/vllm-project/vllm/issues/25194). Keep this one # correctness test on L4 until its H200 output matches the Transformers reference. -- label: Language Models Tests (Granite L4 Compatibility) +- label: ":nvidia: (L4) Granite Language Model Compatibility" key: language-models-tests-granite-l4-compatibility timeout_in_minutes: 65 source_file_dependencies: @@ -97,7 +100,7 @@ steps: - CAUSAL_CONV1D_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0' - pytest -v -s models/language/generation -m hybrid_model -k 'granite-4.0-tiny-preview' -- label: Language Models Test (Extended Generation) # 80min +- label: ":nvidia: (H200) Language Models (Extended Generation)" device: h200_35gb key: language-models-test-extended-generation timeout_in_minutes: 80 @@ -112,7 +115,7 @@ steps: - CAUSAL_CONV1D_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0' - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)' -- label: Language Models Test (PPL) +- label: ":nvidia: (H200) Language Models (PPL)" key: language-models-test-ppl timeout_in_minutes: 30 device: h200_18gb @@ -124,7 +127,7 @@ steps: commands: - pytest -v -s models/language/generation_ppl_test -- label: Language Models Test (Extended Pooling) %N +- label: ":nvidia: (H200) Language Models (Extended Pooling) Shard %N" device: h200_35gb key: language-models-test-extended-pooling timeout_in_minutes: 120 @@ -138,6 +141,7 @@ steps: - pytest -v -s models/language/pooling -m 'not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB mirror: amd: + label: ":amd: (MI300) Language Models (Extended Pooling) Shard %N" dind: false device: mi300_1 timeout_in_minutes: 95 @@ -147,7 +151,7 @@ steps: commands: - pytest -v -s models/language/pooling -m 'not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB -- label: Language Models Test (MTEB) +- label: ":nvidia: (H200) Language Models (MTEB)" key: language-models-test-mteb timeout_in_minutes: 68 device: h200_18gb diff --git a/.buildkite/test_areas/models_multimodal.yaml b/.buildkite/test_areas/models_multimodal.yaml index 2ea3621342..1d97c3f5e6 100644 --- a/.buildkite/test_areas/models_multimodal.yaml +++ b/.buildkite/test_areas/models_multimodal.yaml @@ -2,7 +2,7 @@ group: Models - Multimodal depends_on: - image-build steps: -- label: "Multi-Modal Models (Standard) 1: qwen2" +- label: ":nvidia: (H200) Multimodal Models (Standard) 1: qwen2" key: multi-modal-models-standard-1-qwen2 timeout_in_minutes: 68 device: h200_18gb @@ -15,13 +15,14 @@ steps: - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model mirror: amd: + label: ":amd: (MI300) Multimodal Models (Standard) 1: qwen2" dind: false device: mi300_1 timeout_in_minutes: 65 depends_on: - image-build-amd -- label: "Multi-Modal Models (Standard) 2: qwen3 + gemma" +- label: ":nvidia: (H200) Multimodal Models (Standard) 2: qwen3 + gemma" key: multi-modal-models-standard-2-qwen3-gemma timeout_in_minutes: 75 device: h200_18gb @@ -35,13 +36,14 @@ steps: - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model mirror: amd: + label: ":amd: (MI300) Multimodal Models (Standard) 2: qwen3 + gemma" dind: false device: mi300_1 timeout_in_minutes: 55 depends_on: - image-build-amd -- label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl" +- label: ":nvidia: (H200) Multimodal Models (Standard) 3: llava + qwen2_vl" device: h200_35gb key: multi-modal-models-standard-3-llava-qwen2-vl timeout_in_minutes: 40 @@ -54,12 +56,13 @@ steps: - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model mirror: amd: + label: ":amd: (MI250) Multimodal Models (Standard) 3: llava + qwen2_vl" device: mi250_1 timeout_in_minutes: 55 depends_on: - image-build-amd -- label: "Multi-Modal Models (Standard) 4: other + whisper" +- label: ":nvidia: (H200) Multimodal Models (Standard) 4: other + whisper" device: h200_35gb key: multi-modal-models-standard-4-other-whisper timeout_in_minutes: 75 @@ -74,13 +77,14 @@ steps: - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model # Otherwise, mp_method="spawn" doesn't work mirror: amd: + label: ":amd: (MI300) Multimodal Models (Standard) 4: other + whisper" dind: false device: mi300_1 timeout_in_minutes: 50 depends_on: - image-build-amd -- label: Multi-Modal Processor (CPU) %N +- label: ":computer: (CPU) Multimodal Processor Shard %N" key: multi-modal-processor-cpu depends_on: - image-build-cpu @@ -95,7 +99,7 @@ steps: - pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB parallelism: 4 -- label: Multi-Modal Processor # 44min +- label: ":nvidia: (H200) Multimodal Processor" key: multi-modal-processor timeout_in_minutes: 98 device: h200_18gb @@ -107,7 +111,7 @@ steps: commands: - pytest -v -s models/multimodal/processing/test_tensor_schema.py -- label: Multi-Modal Accuracy Eval (Small Models) # 50min +- label: ":nvidia: (H200) Multimodal Accuracy Eval (Small Models)" device: h200_35gb key: multi-modal-accuracy-eval-small-models timeout_in_minutes: 30 @@ -120,6 +124,7 @@ steps: - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1 mirror: amd: + label: ":amd: (MI300) Multimodal Accuracy Eval (Small Models)" dind: false device: mi300_1 timeout_in_minutes: 35 @@ -132,7 +137,7 @@ steps: - vllm/platforms/rocm.py - vllm/model_executor/model_loader/ -- label: Multi-Modal Models (Extended Generation 1) +- label: ":nvidia: (H200) Multimodal Models (Extended Generation 1)" device: h200_35gb key: multi-modal-models-extended-generation-1 optional: true @@ -146,13 +151,14 @@ steps: - pytest -v -s models/multimodal/test_mapping.py mirror: amd: + label: ":amd: (MI300) Multimodal Models (Extended Generation 1)" dind: false device: mi300_1 timeout_in_minutes: 90 depends_on: - image-build-amd -- label: Multi-Modal Models (PPL) +- label: ":nvidia: (H200) Multimodal Models (PPL)" device: h200_35gb key: multi-modal-models-extended-ppl optional: true @@ -163,13 +169,14 @@ steps: - pytest -v -s models/multimodal/generation_ppl_test/ mirror: amd: + label: ":amd: (MI300) Multimodal Models (PPL)" dind: false device: mi300_1 timeout_in_minutes: 90 depends_on: - image-build-amd -- label: Multi-Modal Models (Extended Generation 2) %N +- label: ":nvidia: (H200) Multimodal Models (Extended Generation 2) Shard %N" device: h200_35gb key: multi-modal-models-extended-generation-2 parallelism: 4 @@ -181,7 +188,7 @@ steps: commands: - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB -- label: Multi-Modal Models (Extended Generation 3) +- label: ":nvidia: (H200) Multimodal Models (Extended Generation 3)" device: h200_35gb key: multi-modal-models-extended-generation-3 optional: true @@ -192,7 +199,7 @@ steps: commands: - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model' -- label: Multi-Modal Models (Extended Pooling) +- label: ":nvidia: (H200) Multimodal Models (Extended Pooling)" key: multi-modal-models-extended-pooling optional: true device: h200_18gb @@ -204,6 +211,7 @@ steps: - pytest -v -s models/multimodal/pooling -m 'not core_model' mirror: amd: + label: ":amd: (MI300) Multimodal Models (Extended Pooling)" dind: false device: mi300_1 timeout_in_minutes: 60 diff --git a/.buildkite/test_areas/plugins.yaml b/.buildkite/test_areas/plugins.yaml index 4ba3392071..2cc4cb8e8b 100644 --- a/.buildkite/test_areas/plugins.yaml +++ b/.buildkite/test_areas/plugins.yaml @@ -2,7 +2,7 @@ group: Plugins depends_on: - image-build steps: -- label: Plugin Tests (2 GPUs) +- label: ":nvidia: (L4) Plugin Integration" key: plugin-tests-2-gpus timeout_in_minutes: 35 working_dir: "/vllm-workspace/tests" @@ -51,7 +51,7 @@ steps: - pytest -v -s plugins_tests/lora_resolvers # unit tests for in-tree lora resolver plugins -- label: GGUF Plugin +- label: ":nvidia: (H200) GGUF Plugin" key: gguf-plugin device: h200_18gb timeout_in_minutes: 30 @@ -64,7 +64,7 @@ steps: - pip install "vllm-gguf-plugin >= 0.0.2" - pytest -v -s plugins_tests/gguf -- label: BitsAndBytes Plugin +- label: ":nvidia: (H200) BitsAndBytes Plugin" key: bitsandbytes-plugin device: h200_18gb timeout_in_minutes: 30 @@ -81,7 +81,7 @@ steps: - pip install "vllm-bnb-plugin >= 0.0.1" - pytest -v -s plugins_tests/bitsandbytes -m 'not distributed' -- label: BitsAndBytes Plugin (2 GPUs) +- label: ":nvidia: (L4) BitsAndBytes Plugin" key: bitsandbytes-plugin-2-gpus timeout_in_minutes: 15 working_dir: "/vllm-workspace/tests" diff --git a/.buildkite/test_areas/pytorch.yaml b/.buildkite/test_areas/pytorch.yaml index b119e981f0..78cebb2e59 100644 --- a/.buildkite/test_areas/pytorch.yaml +++ b/.buildkite/test_areas/pytorch.yaml @@ -2,7 +2,7 @@ group: PyTorch depends_on: - image-build steps: -- label: PyTorch Compilation Unit Tests +- label: ":nvidia: (H200) PyTorch Compilation" device: h200_35gb key: pytorch-compilation-unit-tests timeout_in_minutes: 60 @@ -43,7 +43,7 @@ steps: # (using -0 for proper path handling) - "find compile/ -maxdepth 1 -name 'test_*.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'" -- label: PyTorch Compilation Unit Tests (H100) +- label: ":nvidia: (H200) PyTorch Compilation H100 Cases" key: pytorch-compilation-unit-tests-h100 timeout_in_minutes: 30 device: h200_18gb @@ -77,7 +77,7 @@ steps: commands: - "find compile/h100/ -name 'test_*.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'" -- label: PyTorch Compilation Passes Unit Tests +- label: ":nvidia: (L4) PyTorch Compilation Passes" key: pytorch-compilation-passes-unit-tests timeout_in_minutes: 45 source_file_dependencies: @@ -110,7 +110,7 @@ steps: commands: - pytest -s -v compile/passes --ignore compile/passes/distributed -- label: PyTorch Fullgraph Test +- label: ":nvidia: (H200) PyTorch Fullgraph" device: h200_35gb key: pytorch-fullgraph-test timeout_in_minutes: 90 @@ -151,7 +151,7 @@ steps: # Hopper-only DeepSeek-V2-Lite cases in this file require two 29.3-GiB model # instances and cannot fit a 35GB MIG slice. L4 retains the original coverage: # those SM90 cases skip while the architecture-compatible cases still run. -- label: PyTorch Fullgraph CUDAGraph (L4 Compatibility) +- label: ":nvidia: (L4) PyTorch Fullgraph CUDAGraph Compatibility" key: pytorch-fullgraph-cudagraph-l4-compatibility timeout_in_minutes: 60 source_file_dependencies: @@ -184,7 +184,7 @@ steps: commands: - pytest -s -v compile/fullgraph/test_full_cudagraph.py -- label: Pytorch Nightly Dependency Override Check # 2min +- label: ":nvidia: (H200) PyTorch Nightly Dependency Override Check" key: pytorch-nightly-dependency-override-check # if this test fails, it means the nightly torch version is not compatible with some # of the dependencies. Please check the error message and add the package to whitelist @@ -197,6 +197,7 @@ steps: - bash standalone_tests/pytorch_nightly_dependency.sh mirror: amd: + label: ":amd: (MI300) PyTorch Nightly Dependency Override Check" dind: false device: mi300_1 timeout_in_minutes: 30 diff --git a/.buildkite/test_areas/quantization.yaml b/.buildkite/test_areas/quantization.yaml index b5c7c7a66d..e659cd75f6 100644 --- a/.buildkite/test_areas/quantization.yaml +++ b/.buildkite/test_areas/quantization.yaml @@ -2,7 +2,7 @@ group: Quantization depends_on: - image-build steps: -- label: Quantization %N +- label: ":nvidia: (H200) Quantization Shard %N" device: h200_35gb key: quantization timeout_in_minutes: 40 @@ -21,7 +21,7 @@ steps: # parameter. It was not exercised by the previous L4 job. - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py -k 'not test_compressed_tensors_w4a8_fp8' --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT -- label: Quantized Fusions +- label: ":nvidia: (H200) Quantized Fusions" device: h200_35gb key: quantized-fusions timeout_in_minutes: 20 @@ -34,7 +34,7 @@ steps: commands: - pytest -v -s fusion/ -- label: Quantized MoE Test (B200) +- label: ":nvidia: (B200) Quantized MoE" key: quantized-moe-test-b200 timeout_in_minutes: 120 working_dir: "/vllm-workspace/" @@ -52,7 +52,7 @@ steps: commands: - pytest -s -v tests/quantization/test_blackwell_moe.py -- label: Quantized Models Test +- label: ":nvidia: (H200) Quantized Models" device: h200_35gb key: quantized-models-test timeout_in_minutes: 65 diff --git a/.buildkite/test_areas/ray_compat.yaml b/.buildkite/test_areas/ray_compat.yaml index 9207621a58..243db5f211 100644 --- a/.buildkite/test_areas/ray_compat.yaml +++ b/.buildkite/test_areas/ray_compat.yaml @@ -2,7 +2,7 @@ group: Ray Compatibility depends_on: - image-build steps: -- label: Ray Dependency Compatibility Check +- label: ":nvidia: (H200) Ray Dependency Compatibility Check" key: ray-dependency-compatibility-check # Informational only — does not block the pipeline. # If this fails, it means the PR introduces a dependency that diff --git a/.buildkite/test_areas/rust_frontend.yaml b/.buildkite/test_areas/rust_frontend.yaml index d562716701..36b7d375d4 100644 --- a/.buildkite/test_areas/rust_frontend.yaml +++ b/.buildkite/test_areas/rust_frontend.yaml @@ -2,7 +2,7 @@ group: Rust Frontend E2E depends_on: - image-build steps: -- label: Rust Frontend OpenAI Coverage +- label: ":nvidia: (H200) Rust Frontend OpenAI Coverage" timeout_in_minutes: 30 device: h200_18gb working_dir: "/vllm-workspace/tests" @@ -37,7 +37,7 @@ steps: - pytest -v -s entrypoints/openai/test_uds.py - pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server" -- label: Rust Frontend Serve/Admin Coverage +- label: ":nvidia: (H200) Rust Frontend Serve/Admin Coverage" timeout_in_minutes: 25 device: h200_18gb working_dir: "/vllm-workspace/tests" @@ -66,7 +66,7 @@ steps: # /tokenizer_info is not implemented in the Rust frontend (the CLI flag is accepted as a no-op). - pytest -v -s entrypoints/serve/tokenize/test_tokenization.py -k "not tokenizer_info" -- label: Rust Frontend Core Correctness +- label: ":nvidia: (H200) Rust Frontend Core Correctness" timeout_in_minutes: 20 device: h200_18gb working_dir: "/vllm-workspace/tests" @@ -80,7 +80,7 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine -- label: Rust Frontend Tool Use +- label: ":nvidia: (H200) Rust Frontend Tool Use" device: h200_35gb timeout_in_minutes: 25 working_dir: "/vllm-workspace/tests" @@ -95,7 +95,7 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s tool_use --ignore=tool_use/mistral --models llama3.2 -k "not test_response_format_with_tool_choice_required and not test_parallel_tool_calls_false and not test_tool_call_and_choice" -- label: Rust Frontend Distributed +- label: ":nvidia: (L4) Rust Frontend Distributed" timeout_in_minutes: 25 num_devices: 4 working_dir: "/vllm-workspace/tests" diff --git a/.buildkite/test_areas/rust_frontend_cargo.yaml b/.buildkite/test_areas/rust_frontend_cargo.yaml index 21d4c2ac21..11bca62526 100644 --- a/.buildkite/test_areas/rust_frontend_cargo.yaml +++ b/.buildkite/test_areas/rust_frontend_cargo.yaml @@ -1,7 +1,7 @@ group: Rust Frontend Cargo depends_on: [] steps: -- label: Rust Frontend Cargo Style + Clippy +- label: ":computer: (CPU) Rust Frontend Cargo Style + Clippy" key: rust-frontend-cargo-style-clippy depends_on: [] timeout_in_minutes: 20 @@ -15,7 +15,7 @@ steps: commands: - .buildkite/scripts/run-rust-frontend-cargo-ci.sh style-clippy -- label: Rust Frontend Cargo Tests +- label: ":computer: (CPU) Rust Frontend Cargo" key: rust-frontend-cargo-tests depends_on: [] timeout_in_minutes: 20 diff --git a/.buildkite/test_areas/samplers.yaml b/.buildkite/test_areas/samplers.yaml index 929cbec2ae..c0297d79e5 100644 --- a/.buildkite/test_areas/samplers.yaml +++ b/.buildkite/test_areas/samplers.yaml @@ -2,7 +2,7 @@ group: Samplers depends_on: - image-build steps: -- label: Samplers Test +- label: ":nvidia: (H200) Samplers" device: h200_35gb key: samplers-test timeout_in_minutes: 40 @@ -19,6 +19,7 @@ steps: - VLLM_USE_FLASHINFER_SAMPLER=1 pytest -v -s samplers mirror: amd: + label: ":amd: (MI250) Samplers" device: mi250_1 timeout_in_minutes: 40 depends_on: diff --git a/.buildkite/test_areas/spec_decode.yaml b/.buildkite/test_areas/spec_decode.yaml index 852e5a4634..7486a467fb 100644 --- a/.buildkite/test_areas/spec_decode.yaml +++ b/.buildkite/test_areas/spec_decode.yaml @@ -2,7 +2,7 @@ group: Spec Decode depends_on: - image-build steps: -- label: V1 Spec Decode +- label: ":nvidia: (H200) V1 Spec Decode" device: h200_35gb key: v1-spec-decode timeout_in_minutes: 40 @@ -23,13 +23,14 @@ steps: - pytest -v -s -m 'not slow_test' v1/spec_decode mirror: amd: + label: ":amd: (MI300) V1 Spec Decode" dind: false device: mi300_1 timeout_in_minutes: 50 depends_on: - image-build-amd -- label: "Spec Decode Eagle 1: DeepSeek + Qwen" +- label: ":nvidia: (H200) Spec Decode Eagle 1: DeepSeek + Qwen" key: spec-decode-eagle-1-deepseek-qwen timeout_in_minutes: 25 device: h200_35gb @@ -41,6 +42,7 @@ steps: - pytest -v -s v1/e2e/spec_decode/eagle/ -k "deepseek_eagle or qwen3_eagle3" mirror: amd: + label: ":amd: (MI300) Spec Decode Eagle 1: DeepSeek + Qwen" dind: false device: mi300_1 timeout_in_minutes: 25 @@ -55,7 +57,7 @@ steps: - tests/v1/e2e/spec_decode/ - vllm/platforms/rocm.py -- label: "Spec Decode Eagle 2: Llama 3 + Qwen VL + Other" +- label: ":nvidia: (H200) Spec Decode Eagle 2: Llama 3 + Qwen VL + Other" key: spec-decode-eagle-2-llama3-qwen-vl-other timeout_in_minutes: 25 device: h200_35gb @@ -67,6 +69,7 @@ steps: - pytest -v -s v1/e2e/spec_decode/eagle/ -k "not deepseek_eagle and not qwen3_eagle3" mirror: amd: + label: ":amd: (MI300) Spec Decode Eagle 2: Llama 3 + Qwen VL + Other" dind: false device: mi300_1 timeout_in_minutes: 25 @@ -81,7 +84,7 @@ steps: - tests/v1/e2e/spec_decode/ - vllm/platforms/rocm.py -- label: Spec Decode Eagle Nightly (B200) +- label: ":nvidia: (B200) Spec Decode Eagle Nightly" key: spec-decode-eagle-nightly-b200 timeout_in_minutes: 25 device: b200-k8s @@ -93,7 +96,7 @@ steps: commands: - pytest -v -s v1/e2e/spec_decode/eagle/ -- label: Spec Decode Speculators + MTP +- label: ":nvidia: (H200) Spec Decode Speculators + MTP" key: spec-decode-speculators-mtp timeout_in_minutes: 50 device: h200_35gb @@ -108,6 +111,7 @@ steps: - pytest -v -s v1/e2e/spec_decode/mtp/ mirror: amd: + label: ":amd: (MI300) Spec Decode Speculators + MTP" dind: false device: mi300_1 timeout_in_minutes: 75 @@ -123,7 +127,7 @@ steps: - tests/v1/e2e/spec_decode/ - vllm/platforms/rocm.py -- label: Spec Decode Speculators + MTP Nightly (B200) +- label: ":nvidia: (B200) Spec Decode Speculators + MTP Nightly" key: spec-decode-speculators-mtp-nightly-b200 timeout_in_minutes: 30 device: b200-k8s @@ -137,7 +141,7 @@ steps: - pytest -v -s v1/e2e/spec_decode/speculators/ - pytest -v -s v1/e2e/spec_decode/mtp/ -- label: Spec Decode Ngram + Suffix +- label: ":nvidia: (H200) Spec Decode N-Gram + Suffix" key: spec-decode-ngram-suffix timeout_in_minutes: 20 device: h200_35gb @@ -151,6 +155,7 @@ steps: - python3 spec_decode/test_custom_proposer.py mirror: amd: + label: ":amd: (MI300) Spec Decode N-Gram + Suffix" dind: false device: mi300_1 timeout_in_minutes: 35 @@ -165,7 +170,7 @@ steps: - tests/v1/e2e/spec_decode/ - vllm/platforms/rocm.py -- label: Spec Decode Draft Model +- label: ":nvidia: (H200) Spec Decode Draft Model" key: spec-decode-draft-model timeout_in_minutes: 45 device: h200_18gb @@ -177,6 +182,7 @@ steps: - pytest -v -s v1/e2e/spec_decode/draft_model/ mirror: amd: + label: ":amd: (MI300) Spec Decode Draft Model" dind: false device: mi300_1 timeout_in_minutes: 55 @@ -191,7 +197,7 @@ steps: - tests/v1/e2e/spec_decode/ - vllm/platforms/rocm.py -- label: Spec Decode Draft Model Nightly (B200) +- label: ":nvidia: (B200) Spec Decode Draft Model Nightly" key: spec-decode-draft-model-nightly-b200 timeout_in_minutes: 40 device: b200-k8s @@ -203,7 +209,7 @@ steps: commands: - pytest -v -s v1/e2e/spec_decode/draft_model/ -- label: Speculators Correctness Nightly (H100) +- label: ":nvidia: (H100) Speculators Correctness Nightly" key: speculators-correctness timeout_in_minutes: 30 device: h100 @@ -218,7 +224,7 @@ steps: - export VLLM_ALLOW_INSECURE_SERIALIZATION=1 - pytest -v -s v1/spec_decode/test_speculators_correctness.py -m slow_test -- label: Spec Decode DeepSeek MTP Parallel Load (2xB200-2xMI300) +- label: ":nvidia: (B200) Spec Decode DeepSeek MTP Parallel Load" key: spec-decode-deepseek-mtp-parallel-load-2xb200-2xmi300 timeout_in_minutes: 30 device: b200-k8s @@ -235,6 +241,7 @@ steps: - pytest -v -s v1/e2e/spec_decode/test_mtp_parallel_load.py mirror: amd: + label: ":amd: (MI300) Spec Decode DeepSeek MTP Parallel Load" dind: false device: mi300_2 timeout_in_minutes: 45 @@ -254,7 +261,7 @@ steps: - tests/v1/e2e/spec_decode/test_mtp_parallel_load.py - vllm/platforms/rocm.py -- label: Spec Decode AL DFlash Nightly +- label: ":nvidia: (H200) Spec Decode AL DFlash Nightly" key: spec-decode-dflash-nightly timeout_in_minutes: 90 device: h200_35gb @@ -269,6 +276,7 @@ steps: - pytest -v -s v1/e2e/spec_decode/acceptance_rates/dflash/ mirror: amd: + label: ":amd: (MI300) Spec Decode AL DFlash Nightly" dind: false device: mi300_1 timeout_in_minutes: 120 @@ -289,7 +297,7 @@ steps: - tests/v1/e2e/spec_decode/ - vllm/platforms/rocm.py -- label: Spec Decode AL DSpark Nightly +- label: ":nvidia: (H200) Spec Decode AL DSpark Nightly" key: spec-decode-dspark-nightly timeout_in_minutes: 60 device: h200_35gb @@ -304,6 +312,7 @@ steps: - pytest -v -s v1/e2e/spec_decode/acceptance_rates/dspark/ mirror: amd: + label: ":amd: (MI300) Spec Decode AL DSpark Nightly" dind: false device: mi300_1 timeout_in_minutes: 90 @@ -328,7 +337,7 @@ steps: - tests/v1/e2e/spec_decode/ - vllm/platforms/rocm.py -- label: Spec Decode AL MTP + Other Acceptance Nightly +- label: ":nvidia: (H200) Spec Decode AL MTP + Other Acceptance Nightly" key: spec-decode-mtp-other-acceptance-nightly timeout_in_minutes: 60 device: h200_35gb diff --git a/.buildkite/test_areas/torch_abi.yaml b/.buildkite/test_areas/torch_abi.yaml index eaef355166..9e8375587a 100644 --- a/.buildkite/test_areas/torch_abi.yaml +++ b/.buildkite/test_areas/torch_abi.yaml @@ -2,7 +2,7 @@ group: Torch ABI depends_on: - image-build steps: -- label: Torch Stable ABI Audit +- label: ":nvidia: (L4) Torch Stable ABI Audit" key: torch-stable-abi-audit timeout_in_minutes: 5 source_file_dependencies: diff --git a/.buildkite/test_areas/weight_loading.yaml b/.buildkite/test_areas/weight_loading.yaml index f587b6210e..0aa25dbad4 100644 --- a/.buildkite/test_areas/weight_loading.yaml +++ b/.buildkite/test_areas/weight_loading.yaml @@ -2,7 +2,7 @@ group: Weight Loading depends_on: - image-build steps: -- label: Weight Loading Multiple GPU # 33min +- label: ":nvidia: (L4) Weight Loading Multi-GPU" key: weight-loading-multiple-gpu timeout_in_minutes: 50 working_dir: "/vllm-workspace/tests" @@ -16,6 +16,7 @@ steps: - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models.txt mirror: amd: + label: ":amd: (MI300) Weight Loading Multi-GPU" dind: false device: mi300_2 timeout_in_minutes: 35