[ROCm][CI] Stage G gating (#50922)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
8d16ca6cc2
commit
e581e14002
@@ -34,3 +34,18 @@ steps:
|
||||
- vllm/v1/attention/
|
||||
commands:
|
||||
- python3 benchmarks/attention_benchmarks/benchmark.py --backends flash flashinfer --batch-specs "8q1s1k"
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) Attention Benchmark Smoke"
|
||||
dind: false
|
||||
device: mi355_2
|
||||
timeout_in_minutes: 25
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- benchmarks/attention_benchmarks/
|
||||
- vllm/v1/attention/
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- python3 benchmarks/attention_benchmarks/benchmark.py --backends ROCM_ATTN ROCM_AITER_FA ROCM_AITER_UNIFIED_ATTN --batch-specs "8q1s1k"
|
||||
|
||||
@@ -205,6 +205,25 @@ steps:
|
||||
commands:
|
||||
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
|
||||
- bash v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) MultiConnector (Nixl+Offloading) PD accuracy"
|
||||
dind: false
|
||||
device: mi355_2
|
||||
timeout_in_minutes: 40
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/offloading/
|
||||
- tests/v1/kv_connector/nixl_integration/
|
||||
- requirements/kv_connectors_rocm.txt
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
|
||||
- ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh
|
||||
|
||||
- label: ":nvidia: (A100) NixlConnector PD + Spec Decode acceptance"
|
||||
key: nixlconnector-pd-spec-decode-acceptance-2-gpus
|
||||
|
||||
@@ -219,6 +219,41 @@ steps:
|
||||
# default does not fit an L4.
|
||||
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
|
||||
- RDT_MODEL=hmellor/tiny-random-DeepseekV2ForCausalLM python3 examples/rl/rlhf_sharded_rdt_small_ep.py
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) Distributed Torchrun + Examples"
|
||||
dind: false
|
||||
device: mi355_4
|
||||
timeout_in_minutes: 45
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/
|
||||
- "!vllm/distributed/kv_transfer/"
|
||||
- tests/distributed/test_torchrun_example.py
|
||||
- tests/distributed/test_torchrun_example_moe.py
|
||||
- examples/rl/
|
||||
- examples/features/data_parallel/data_parallel_offline.py
|
||||
- .buildkite/scripts/install-kv-connectors.sh
|
||||
- requirements/kv_connectors_rocm.txt
|
||||
- requirements/test/rocm.in
|
||||
- requirements/test/rocm.txt
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
# Sharded RDT needs Ray >= 2.56; keep the ROCm image's dependencies.
|
||||
- uv pip install --system --no-deps 'ray==2.56.1'
|
||||
- export NCCL_CUMEM_HOST_ENABLE=0
|
||||
- torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example.py
|
||||
- PP_SIZE=2 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example.py
|
||||
- TP_SIZE=4 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
|
||||
- PP_SIZE=2 TP_SIZE=2 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
|
||||
- DP_SIZE=4 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
|
||||
- TP_SIZE=2 DP_SIZE=2 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
|
||||
- python3 examples/features/data_parallel/data_parallel_offline.py --enforce-eager
|
||||
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_nccl.py
|
||||
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_ipc.py
|
||||
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
|
||||
- RDT_MODEL=hmellor/tiny-random-DeepseekV2ForCausalLM python3 examples/rl/rlhf_sharded_rdt_small_ep.py
|
||||
|
||||
- label: ":nvidia: (L4) Distributed DP Extended"
|
||||
key: distributed-dp-tests-4-gpus
|
||||
@@ -291,6 +326,23 @@ steps:
|
||||
- export NCCL_CUMEM_HOST_ENABLE=0
|
||||
# test with torchrun tp=2 and dp=4 with ep
|
||||
- torchrun --nproc-per-node=8 ../examples/features/torchrun/torchrun_dp_example_offline.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) Distributed DP + EP"
|
||||
dind: false
|
||||
device: mi355_8
|
||||
timeout_in_minutes: 50
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- examples/features/torchrun/torchrun_dp_example_offline.py
|
||||
- vllm/config/parallel.py
|
||||
- vllm/distributed/
|
||||
- "!vllm/distributed/kv_transfer/"
|
||||
- vllm/v1/engine/llm_engine.py
|
||||
- vllm/v1/executor/uniproc_executor.py
|
||||
- vllm/v1/worker/gpu_worker.py
|
||||
- vllm/platforms/rocm.py
|
||||
|
||||
- label: ":nvidia: (A100) Distributed"
|
||||
key: distributed-tests-4xa100
|
||||
|
||||
@@ -11,6 +11,28 @@ steps:
|
||||
working_dir: "/vllm-workspace"
|
||||
commands:
|
||||
- bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) DeepSeek V2-Lite Sync EPLB Accuracy"
|
||||
dind: false
|
||||
device: mi355_4
|
||||
timeout_in_minutes: 40
|
||||
env:
|
||||
VLLM_ENGINE_READY_TIMEOUT_S: "1800"
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/model_executor/models/
|
||||
- vllm/model_executor/model_loader/
|
||||
- vllm/distributed/eplb
|
||||
- vllm/model_executor/layers/fused_moe/
|
||||
- vllm/model_executor/layers/quantization/
|
||||
- vllm/v1/attention/backends/
|
||||
- vllm/v1/attention/backends/mla/
|
||||
- vllm/v1/attention/selector.py
|
||||
- .buildkite/scripts/scheduled_integration_test/
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
|
||||
- label: ":nvidia: (H100) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy"
|
||||
key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-4xh100
|
||||
@@ -21,6 +43,25 @@ steps:
|
||||
working_dir: "/vllm-workspace"
|
||||
commands:
|
||||
- bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy TP2"
|
||||
dind: false
|
||||
device: mi355_4
|
||||
timeout_in_minutes: 30
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/model_executor/models/
|
||||
- vllm/model_executor/model_loader/
|
||||
- vllm/model_executor/layers/quantization/
|
||||
- vllm/distributed/eplb
|
||||
- vllm/model_executor/layers/fused_moe/
|
||||
- vllm/v1/attention/backends/
|
||||
- vllm/v1/attention/selector.py
|
||||
- .buildkite/scripts/scheduled_integration_test/
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
|
||||
- label: ":nvidia: (B200) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy"
|
||||
key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-2xb200
|
||||
@@ -31,6 +72,25 @@ steps:
|
||||
working_dir: "/vllm-workspace"
|
||||
commands:
|
||||
- bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 2 1
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy TP1"
|
||||
dind: false
|
||||
device: mi355_2
|
||||
timeout_in_minutes: 25
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/model_executor/models/
|
||||
- vllm/model_executor/model_loader/
|
||||
- vllm/model_executor/layers/quantization/
|
||||
- vllm/model_executor/layers/fused_moe/
|
||||
- vllm/distributed/eplb
|
||||
- vllm/v1/attention/backends/
|
||||
- vllm/v1/attention/selector.py
|
||||
- .buildkite/scripts/scheduled_integration_test/
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
|
||||
- label: ":nvidia: (H100) Qwen3-30B-A3B-FP8 DP4 Async EPLB Accuracy"
|
||||
key: qwen3-30b-a3b-fp8-dp4-async-eplb-accuracy
|
||||
@@ -41,6 +101,25 @@ steps:
|
||||
working_dir: "/vllm-workspace"
|
||||
commands:
|
||||
- bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_dp4_async_eplb.sh 0.8 200 8050
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) Qwen3-30B-A3B-FP8 DP4 Async EPLB Accuracy"
|
||||
dind: false
|
||||
device: mi355_4
|
||||
timeout_in_minutes: 30
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/model_executor/models/
|
||||
- vllm/model_executor/model_loader/
|
||||
- vllm/model_executor/layers/quantization/
|
||||
- vllm/distributed/eplb
|
||||
- vllm/model_executor/layers/fused_moe/
|
||||
- vllm/v1/attention/backends/
|
||||
- vllm/v1/attention/selector.py
|
||||
- .buildkite/scripts/scheduled_integration_test/
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
|
||||
- label: ":nvidia: (H100) DeepSeek V2-Lite Prefetch Offload Accuracy"
|
||||
key: deepseek-v2-lite-prefetch-offload-accuracy-h100
|
||||
|
||||
@@ -40,6 +40,19 @@ steps:
|
||||
commands:
|
||||
- pytest -v -s distributed/test_eplb_execute.py
|
||||
- pytest -v -s distributed/test_eplb_spec_decode.py
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) EPLB Execution"
|
||||
dind: false
|
||||
device: mi355_4
|
||||
timeout_in_minutes: 35
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/eplb
|
||||
- tests/distributed/test_eplb_execute.py
|
||||
- tests/distributed/test_eplb_spec_decode.py
|
||||
- vllm/platforms/rocm.py
|
||||
|
||||
- label: ":nvidia: (H100) Elastic EP Scaling"
|
||||
key: elastic-ep-scaling-test
|
||||
|
||||
@@ -114,6 +114,27 @@ steps:
|
||||
commands:
|
||||
- export VLLM_USE_DEEP_GEMM=0 # We found Triton is faster than DeepGEMM for H100
|
||||
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-hopper.txt --tp-size=4
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) LM Eval Large Models FP8"
|
||||
dind: false
|
||||
device: mi355_4
|
||||
timeout_in_minutes: 45
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- csrc/
|
||||
- vllm/model_executor/layers/quantization
|
||||
- vllm/model_executor/models/
|
||||
- vllm/model_executor/model_loader/
|
||||
- vllm/v1/attention/backends/
|
||||
- vllm/v1/attention/selector.py
|
||||
- .buildkite/lm-eval-harness/
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- export VLLM_USE_DEEP_GEMM=0
|
||||
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-tp4.txt --tp-size=4
|
||||
|
||||
- label: ":nvidia: (B200) LM Eval Small Models"
|
||||
key: lm-eval-small-models-1xb200
|
||||
@@ -342,6 +363,33 @@ steps:
|
||||
- vllm/third_party/flash_linear_attention/ops/
|
||||
commands:
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-blackwell.txt
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) LM Eval Qwen3.5 Models"
|
||||
dind: false
|
||||
device: mi355_2
|
||||
timeout_in_minutes: 40
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/model_executor/models/qwen3_5.py
|
||||
- vllm/model_executor/models/qwen3_5_mtp.py
|
||||
- vllm/transformers_utils/configs/qwen3_5.py
|
||||
- vllm/transformers_utils/configs/qwen3_5_moe.py
|
||||
- vllm/model_executor/models/qwen2.py
|
||||
- vllm/model_executor/models/qwen3.py
|
||||
- vllm/model_executor/models/qwen3_next.py
|
||||
- vllm/model_executor/models/qwen3_next_mtp.py
|
||||
- vllm/third_party/flash_linear_attention/ops/
|
||||
- tests/evals/gsm8k/configs/models-qwen35-mi355.txt
|
||||
- tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-DEP2.yaml
|
||||
- tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-MXFP4-AITER-TP2.yaml
|
||||
- tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-MXFP4-EMU-TP2.yaml
|
||||
- tests/evals/gsm8k/test_gsm8k_correctness.py
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-mi355.txt
|
||||
|
||||
- label: ":nvidia: (H200) LM Eval DeepSeek R1 TP"
|
||||
key: lm-eval-deepseek-r1-tp-8xh200
|
||||
|
||||
@@ -204,3 +204,27 @@ steps:
|
||||
- TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py -k "not 4 and not server_info"
|
||||
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py -k "not 4 and not server_info"
|
||||
- TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py -k "not 4 and not server_info"
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) Rust Frontend Distributed"
|
||||
dind: false
|
||||
device: mi355_4
|
||||
timeout_in_minutes: 45
|
||||
env:
|
||||
VLLM_ENGINE_READY_TIMEOUT_S: "1800"
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- rust/
|
||||
- vllm/distributed/
|
||||
- "!vllm/distributed/kv_transfer/"
|
||||
- vllm/engine/
|
||||
- vllm/v1/executor/
|
||||
- vllm/v1/engine/
|
||||
- vllm/v1/worker/
|
||||
- tests/utils.py
|
||||
- tests/v1/distributed/test_dense_dp_world_size.py
|
||||
- tests/v1/distributed/test_external_lb_dp.py
|
||||
- tests/v1/distributed/test_hybrid_lb_dp.py
|
||||
- tests/v1/distributed/test_internal_lb_dp.py
|
||||
- vllm/platforms/rocm.py
|
||||
|
||||
Reference in New Issue
Block a user