[ROCm][CI] Stage G gating (#50922)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Andreas Karatzas
2026-09-22 19:04:54 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent 8d16ca6cc2
commit e581e14002
7 changed files with 250 additions and 0 deletions
+15
View File
@@ -34,3 +34,18 @@ steps:
- vllm/v1/attention/
commands:
- python3 benchmarks/attention_benchmarks/benchmark.py --backends flash flashinfer --batch-specs "8q1s1k"
mirror:
amd:
label: ":amd: (MI355) Attention Benchmark Smoke"
dind: false
device: mi355_2
timeout_in_minutes: 25
depends_on:
- image-build-amd
source_file_dependencies:
- benchmarks/attention_benchmarks/
- vllm/v1/attention/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- python3 benchmarks/attention_benchmarks/benchmark.py --backends ROCM_ATTN ROCM_AITER_FA ROCM_AITER_UNIFIED_ATTN --batch-specs "8q1s1k"
+19
View File
@@ -205,6 +205,25 @@ steps:
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh
mirror:
amd:
label: ":amd: (MI355) MultiConnector (Nixl+Offloading) PD accuracy"
dind: false
device: mi355_2
timeout_in_minutes: 40
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
- vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py
- vllm/distributed/kv_transfer/kv_connector/v1/offloading/
- tests/v1/kv_connector/nixl_integration/
- requirements/kv_connectors_rocm.txt
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh
- label: ":nvidia: (A100) NixlConnector PD + Spec Decode acceptance"
key: nixlconnector-pd-spec-decode-acceptance-2-gpus
+52
View File
@@ -219,6 +219,41 @@ steps:
# default does not fit an L4.
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- RDT_MODEL=hmellor/tiny-random-DeepseekV2ForCausalLM python3 examples/rl/rlhf_sharded_rdt_small_ep.py
mirror:
amd:
label: ":amd: (MI355) Distributed Torchrun + Examples"
dind: false
device: mi355_4
timeout_in_minutes: 45
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/
- "!vllm/distributed/kv_transfer/"
- tests/distributed/test_torchrun_example.py
- tests/distributed/test_torchrun_example_moe.py
- examples/rl/
- examples/features/data_parallel/data_parallel_offline.py
- .buildkite/scripts/install-kv-connectors.sh
- requirements/kv_connectors_rocm.txt
- requirements/test/rocm.in
- requirements/test/rocm.txt
- vllm/platforms/rocm.py
commands:
# Sharded RDT needs Ray >= 2.56; keep the ROCm image's dependencies.
- uv pip install --system --no-deps 'ray==2.56.1'
- export NCCL_CUMEM_HOST_ENABLE=0
- torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example.py
- PP_SIZE=2 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example.py
- TP_SIZE=4 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
- PP_SIZE=2 TP_SIZE=2 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
- DP_SIZE=4 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
- TP_SIZE=2 DP_SIZE=2 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
- python3 examples/features/data_parallel/data_parallel_offline.py --enforce-eager
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_nccl.py
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_ipc.py
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- RDT_MODEL=hmellor/tiny-random-DeepseekV2ForCausalLM python3 examples/rl/rlhf_sharded_rdt_small_ep.py
- label: ":nvidia: (L4) Distributed DP Extended"
key: distributed-dp-tests-4-gpus
@@ -291,6 +326,23 @@ steps:
- export NCCL_CUMEM_HOST_ENABLE=0
# test with torchrun tp=2 and dp=4 with ep
- torchrun --nproc-per-node=8 ../examples/features/torchrun/torchrun_dp_example_offline.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep
mirror:
amd:
label: ":amd: (MI355) Distributed DP + EP"
dind: false
device: mi355_8
timeout_in_minutes: 50
depends_on:
- image-build-amd
source_file_dependencies:
- examples/features/torchrun/torchrun_dp_example_offline.py
- vllm/config/parallel.py
- vllm/distributed/
- "!vllm/distributed/kv_transfer/"
- vllm/v1/engine/llm_engine.py
- vllm/v1/executor/uniproc_executor.py
- vllm/v1/worker/gpu_worker.py
- vllm/platforms/rocm.py
- label: ":nvidia: (A100) Distributed"
key: distributed-tests-4xa100
@@ -11,6 +11,28 @@ steps:
working_dir: "/vllm-workspace"
commands:
- bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010
mirror:
amd:
label: ":amd: (MI355) DeepSeek V2-Lite Sync EPLB Accuracy"
dind: false
device: mi355_4
timeout_in_minutes: 40
env:
VLLM_ENGINE_READY_TIMEOUT_S: "1800"
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/distributed/eplb
- vllm/model_executor/layers/fused_moe/
- vllm/model_executor/layers/quantization/
- vllm/v1/attention/backends/
- vllm/v1/attention/backends/mla/
- vllm/v1/attention/selector.py
- .buildkite/scripts/scheduled_integration_test/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- label: ":nvidia: (H100) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy"
key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-4xh100
@@ -21,6 +43,25 @@ steps:
working_dir: "/vllm-workspace"
commands:
- bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020
mirror:
amd:
label: ":amd: (MI355) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy TP2"
dind: false
device: mi355_4
timeout_in_minutes: 30
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/model_executor/layers/quantization/
- vllm/distributed/eplb
- vllm/model_executor/layers/fused_moe/
- vllm/v1/attention/backends/
- vllm/v1/attention/selector.py
- .buildkite/scripts/scheduled_integration_test/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- label: ":nvidia: (B200) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy"
key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-2xb200
@@ -31,6 +72,25 @@ steps:
working_dir: "/vllm-workspace"
commands:
- bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 2 1
mirror:
amd:
label: ":amd: (MI355) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy TP1"
dind: false
device: mi355_2
timeout_in_minutes: 25
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/model_executor/layers/quantization/
- vllm/model_executor/layers/fused_moe/
- vllm/distributed/eplb
- vllm/v1/attention/backends/
- vllm/v1/attention/selector.py
- .buildkite/scripts/scheduled_integration_test/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- label: ":nvidia: (H100) Qwen3-30B-A3B-FP8 DP4 Async EPLB Accuracy"
key: qwen3-30b-a3b-fp8-dp4-async-eplb-accuracy
@@ -41,6 +101,25 @@ steps:
working_dir: "/vllm-workspace"
commands:
- bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_dp4_async_eplb.sh 0.8 200 8050
mirror:
amd:
label: ":amd: (MI355) Qwen3-30B-A3B-FP8 DP4 Async EPLB Accuracy"
dind: false
device: mi355_4
timeout_in_minutes: 30
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/model_executor/layers/quantization/
- vllm/distributed/eplb
- vllm/model_executor/layers/fused_moe/
- vllm/v1/attention/backends/
- vllm/v1/attention/selector.py
- .buildkite/scripts/scheduled_integration_test/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- label: ":nvidia: (H100) DeepSeek V2-Lite Prefetch Offload Accuracy"
key: deepseek-v2-lite-prefetch-offload-accuracy-h100
@@ -40,6 +40,19 @@ steps:
commands:
- pytest -v -s distributed/test_eplb_execute.py
- pytest -v -s distributed/test_eplb_spec_decode.py
mirror:
amd:
label: ":amd: (MI355) EPLB Execution"
dind: false
device: mi355_4
timeout_in_minutes: 35
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/eplb
- tests/distributed/test_eplb_execute.py
- tests/distributed/test_eplb_spec_decode.py
- vllm/platforms/rocm.py
- label: ":nvidia: (H100) Elastic EP Scaling"
key: elastic-ep-scaling-test
+48
View File
@@ -114,6 +114,27 @@ steps:
commands:
- export VLLM_USE_DEEP_GEMM=0 # We found Triton is faster than DeepGEMM for H100
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-hopper.txt --tp-size=4
mirror:
amd:
label: ":amd: (MI355) LM Eval Large Models FP8"
dind: false
device: mi355_4
timeout_in_minutes: 45
depends_on:
- image-build-amd
source_file_dependencies:
- csrc/
- vllm/model_executor/layers/quantization
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/v1/attention/backends/
- vllm/v1/attention/selector.py
- .buildkite/lm-eval-harness/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- export VLLM_USE_DEEP_GEMM=0
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-tp4.txt --tp-size=4
- label: ":nvidia: (B200) LM Eval Small Models"
key: lm-eval-small-models-1xb200
@@ -342,6 +363,33 @@ steps:
- vllm/third_party/flash_linear_attention/ops/
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-blackwell.txt
mirror:
amd:
label: ":amd: (MI355) LM Eval Qwen3.5 Models"
dind: false
device: mi355_2
timeout_in_minutes: 40
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/model_executor/models/qwen3_5.py
- vllm/model_executor/models/qwen3_5_mtp.py
- vllm/transformers_utils/configs/qwen3_5.py
- vllm/transformers_utils/configs/qwen3_5_moe.py
- vllm/model_executor/models/qwen2.py
- vllm/model_executor/models/qwen3.py
- vllm/model_executor/models/qwen3_next.py
- vllm/model_executor/models/qwen3_next_mtp.py
- vllm/third_party/flash_linear_attention/ops/
- tests/evals/gsm8k/configs/models-qwen35-mi355.txt
- tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-DEP2.yaml
- tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-MXFP4-AITER-TP2.yaml
- tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-MXFP4-EMU-TP2.yaml
- tests/evals/gsm8k/test_gsm8k_correctness.py
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-mi355.txt
- label: ":nvidia: (H200) LM Eval DeepSeek R1 TP"
key: lm-eval-deepseek-r1-tp-8xh200
+24
View File
@@ -204,3 +204,27 @@ steps:
- TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py -k "not 4 and not server_info"
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py -k "not 4 and not server_info"
- TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py -k "not 4 and not server_info"
mirror:
amd:
label: ":amd: (MI355) Rust Frontend Distributed"
dind: false
device: mi355_4
timeout_in_minutes: 45
env:
VLLM_ENGINE_READY_TIMEOUT_S: "1800"
depends_on:
- image-build-amd
source_file_dependencies:
- rust/
- vllm/distributed/
- "!vllm/distributed/kv_transfer/"
- vllm/engine/
- vllm/v1/executor/
- vllm/v1/engine/
- vllm/v1/worker/
- tests/utils.py
- tests/v1/distributed/test_dense_dp_world_size.py
- tests/v1/distributed/test_external_lb_dp.py
- tests/v1/distributed/test_hybrid_lb_dp.py
- tests/v1/distributed/test_internal_lb_dp.py
- vllm/platforms/rocm.py