[ROCm][CI] Extend AMD coverage for distributed, model, and eval tests (#56679)

Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Signed-off-by: Andreas Karatzas <andreas.karatzas@protonmail.com>
Signed-off-by: Stefan Koncarevic <stefan.koncarevic@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Stefan Koncarevic <stefan.koncarevic@amd.com>
This commit is contained in:
Andreas Karatzas
2026-10-02 23:20:51 -05:00
committed by GitHub
co-authored by OpenAI Codex Stefan Koncarevic
parent 28556f8600
commit c28834df00
6 changed files with 449 additions and 0 deletions
+60
View File
@@ -93,6 +93,24 @@ steps:
commands:
- export VLLM_TEST_CLEAN_GPU_MEMORY=1
- pytest -s -v tests/compile/passes/distributed
mirror:
amd:
label: ':amd: (MI300) Distributed Compile'
dind: false
device: mi300_2
timeout_in_minutes: 65
working_dir: /vllm-workspace/
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/compilation/
- vllm/model_executor/layers
- tests/compile/passes/distributed/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- export VLLM_TEST_CLEAN_GPU_MEMORY=1
- pytest -s -v tests/compile/passes/distributed
- label: ":nvidia: (B200) Fusion and Compile"
key: fusion-and-compile-unit-tests-2xb200
@@ -463,3 +481,45 @@ steps:
- amd-smi
- VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_CUSTOM_AR=1 pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py -k 'test_tp2_ar_rms_fp8_fusions and qwen3 and ROCM_AITER_UNIFIED_ATTN and +quant_fp8 and (dynamo_partition or not +rms_norm)'
- pytest -v -s tests/compile/fusions_e2e/test_tp2_rocm_moe.py
- label: ':amd: (MI355) Fusion E2E TP2 AR-RMS (Inductor Partition)'
key: fusion-e2e-tp2-ar-rms-inductor-partition-amd
dind: false
device: mi355_2
num_devices: 2
timeout_in_minutes: 65
working_dir: /vllm-workspace/
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/compilation/
- vllm/model_executor/layers
- tests/compile/fusions_e2e/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- csrc/quantization/
- vllm/v1/attention/
commands:
- export VLLM_TEST_CLEAN_GPU_MEMORY=1
- pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py::test_tp2_ar_rms_fusions -k "inductor_partition"
- label: ':amd: (MI355) Fusion E2E TP2 AR-RMS (Dynamo Partition)'
key: fusion-e2e-tp2-ar-rms-dynamo-partition-amd
dind: false
device: mi355_2
num_devices: 2
timeout_in_minutes: 65
working_dir: /vllm-workspace/
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/compilation/
- vllm/model_executor/layers
- tests/compile/fusions_e2e/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- csrc/quantization/
- vllm/v1/attention/
commands:
- export VLLM_TEST_CLEAN_GPU_MEMORY=1
- pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py::test_tp2_ar_rms_fusions -k "dynamo_partition"
+135
View File
@@ -17,6 +17,24 @@ steps:
- pytest -v -s distributed/test_shm_broadcast.py
- pytest -v -s distributed/test_shm_buffer.py
- pytest -v -s distributed/test_shm_storage.py
mirror:
amd:
label: ':amd: (MI300) Distributed Comm Ops'
dind: false
device: mi300_2
timeout_in_minutes: 30
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed
- tests/distributed
- vllm/platforms/rocm.py
commands:
- pytest -v -s distributed/test_comm_ops.py
- pytest -v -s distributed/test_shm_broadcast.py
- pytest -v -s distributed/test_shm_buffer.py
- pytest -v -s distributed/test_shm_storage.py
- label: ":nvidia: (L4) Sharded RDT Weight Transfer"
key: sharded-rdt-weight-transfer
@@ -176,6 +194,31 @@ steps:
- VLLM_TEST_SAME_HOST=1 VLLM_TEST_WITH_DEFAULT_DEVICE_SET=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s v1/shutdown
- pytest -v -s v1/worker/test_worker_memory_snapshot.py
mirror:
amd:
label: ':amd: (MI300) Distributed Torchrun + Shutdown'
dind: false
device: mi300_2
timeout_in_minutes: 35
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/
- vllm/engine/
- vllm/v1/executor/
- vllm/v1/worker/worker_base.py
- vllm/v1/engine/
- vllm/v1/worker/
- tests/distributed/
- tests/v1/shutdown
- tests/v1/worker/test_worker_memory_snapshot.py
- vllm/platforms/rocm.py
commands:
- VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
- VLLM_TEST_SAME_HOST=1 VLLM_TEST_WITH_DEFAULT_DEVICE_SET=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
- HIP_VISIBLE_DEVICES=0,1 pytest -v -s v1/shutdown
- pytest -v -s v1/worker/test_worker_memory_snapshot.py
- label: ":nvidia: (L4) Distributed Torchrun + Examples"
key: distributed-torchrun-examples-4-gpus
@@ -282,6 +325,31 @@ steps:
- pytest -v -s v1/engine/test_engine_core_client.py::test_kv_cache_events_dp
- pytest -v -s distributed/test_utils.py
- pytest -v -s distributed/test_engram_dp_shard.py
mirror:
amd:
label: ':amd: (MI355) Distributed DP Extended'
dind: false
device: mi355_4
timeout_in_minutes: 60
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/
- tests/v1/distributed
- tests/v1/engine/test_engine_core_client.py
- tests/distributed/test_utils
- vllm/platforms/rocm.py
commands:
- export NCCL_CUMEM_HOST_ENABLE=0
- TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
- TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
- TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
- TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_dense_dp_world_size.py
- TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py
- TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py
- pytest -v -s v1/engine/test_engine_core_client.py::test_kv_cache_events_dp
- pytest -v -s distributed/test_utils.py
- label: ":nvidia: (L4) Distributed Compile + Comm"
key: distributed-compile-comm-4-gpus
@@ -304,6 +372,27 @@ steps:
- pytest -v -s distributed/test_symm_mem_allreduce.py
# test multi-node TP with multiproc executor (simulated on single node)
- pytest -v -s distributed/test_multiproc_executor.py::test_multiproc_executor_multi_node
mirror:
amd:
label: ':amd: (MI355) Distributed Compile + Comm'
dind: false
device: mi355_4
timeout_in_minutes: 45
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/
- tests/distributed/test_pynccl
- tests/distributed/test_events
- tests/distributed/test_symm_mem_allreduce.py
- tests/distributed/test_multiproc_executor.py
- vllm/platforms/rocm.py
commands:
- pytest -v -s distributed/test_pynccl.py
- pytest -v -s distributed/test_events.py
- pytest -v -s distributed/test_symm_mem_allreduce.py
- pytest -v -s distributed/test_multiproc_executor.py::test_multiproc_executor_multi_node
- label: ":nvidia: (H100) Distributed DP + EP"
key: distributed-tests-8xh100
@@ -359,6 +448,28 @@ steps:
- torchrun --nproc_per_node=2 distributed/test_ca_buffer_sharing.py
- TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
- pytest -v -s -x lora/test_mixtral.py
mirror:
amd:
label: ':amd: (MI355) Distributed'
dind: false
device: mi355_4
timeout_in_minutes: 40
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/
- tests/models/test_vision.py
- vllm/platforms/rocm.py
- tests/distributed/
- tests/basic_correctness/
- tests/lora/test_mixtral.py
commands:
- pytest -v -s distributed/test_custom_all_reduce.py
- torchrun --nproc_per_node=2 distributed/test_ca_buffer_sharing.py
- TARGET_TEST_SUITE=MI355 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
- pytest -v -s -x lora/test_mixtral.py
- pytest -v -s models/test_vision.py -m 'distributed(num_gpus=4)'
- label: ":nvidia: (H100) Distributed Features"
key: distributed-tests-2xh100-2xmi300
@@ -583,3 +694,27 @@ steps:
- pytest -v -s distributed/test_ray_v2_executor_e2e.py
- pytest -v -s distributed/test_pipeline_parallel.py -k "ray"
- TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/models/test_basic_correctness.py -k "ray"
mirror:
amd:
label: ':amd: (MI355) RayExecutorV2'
dind: false
device: mi355_4
timeout_in_minutes: 75
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/v1/executor/ray_executor_v2.py
- vllm/v1/executor/abstract.py
- vllm/v1/executor/multiproc_executor.py
- tests/distributed/test_ray_v2_executor.py
- tests/distributed/test_ray_v2_executor_e2e.py
- tests/distributed/test_pipeline_parallel.py
- tests/basic_correctness/models/test_basic_correctness.py
- vllm/platforms/rocm.py
commands:
- export VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1
- pytest -v -s distributed/test_ray_v2_executor.py
- pytest -v -s distributed/test_ray_v2_executor_e2e.py
- pytest -v -s distributed/test_pipeline_parallel.py -k "ray"
- TARGET_TEST_SUITE=MI355 pytest -v -s basic_correctness/models/test_basic_correctness.py -k "ray"
@@ -150,3 +150,27 @@ steps:
- .buildkite/scripts/scheduled_integration_test/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- label: ":amd: (MI355) Qwen3-Next-80B-A3B-Instruct MTP Async EPLB Accuracy"
key: amd-qwen3-next-mtp-async-eplb-accuracy
dind: false
device: mi355_4
timeout_in_minutes: 45
depends_on:
- image-build-amd
working_dir: /vllm-workspace
source_file_dependencies:
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/v1/spec_decode/
- vllm/distributed/eplb
- vllm/model_executor/layers/fused_moe/
- vllm/model_executor/layers/quantization/
- vllm/v1/attention/backends/
- vllm/v1/attention/selector.py
- .buildkite/scripts/scheduled_integration_test/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
num_devices: 4
commands:
- VLLM_ENGINE_READY_TIMEOUT_S=1800 bash .buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh 0.8 1319 8040
+147
View File
@@ -753,6 +753,32 @@ steps:
parallelism: 4
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor/config-b200-shard-$$BUILDKITE_PARALLEL_JOB.txt
mirror:
amd:
label: ":amd: (MI355) MoE Refactor BF16 Integration"
dind: false
device: mi355_2
num_devices: 2
parallelism: 1
timeout_in_minutes: 105
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- tests/evals/gsm8k/
- tests/utils.py
- vllm/model_executor/layers/fused_moe/
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/model_executor/kernels/linear/
- vllm/v1/attention/
- vllm/distributed/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
env:
VLLM_WORKER_MULTIPROC_METHOD: "spawn"
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor/config-b200.txt -k "Llama-4-Scout-BF16-triton or Mixtral-8x7B-BF16-triton"
- label: ":nvidia: (B200) MoE Refactor DP Integration TEMPORARY"
key: moe-refactor-integration-test-b200-dp-temporary
@@ -761,6 +787,32 @@ steps:
num_devices: 2
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt
mirror:
amd:
label: ":amd: (MI355) MoE Refactor BF16 DP Integration"
dind: false
device: mi355_2
num_devices: 2
timeout_in_minutes: 105
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- tests/evals/gsm8k/
- tests/utils.py
- vllm/config/parallel.py
- vllm/model_executor/layers/fused_moe/
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/model_executor/kernels/linear/
- vllm/v1/attention/
- vllm/distributed/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
env:
VLLM_WORKER_MULTIPROC_METHOD: "spawn"
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt -k "Qwen3-30B-A3B-BF16-triton"
- label: ":nvidia: (L4) LM Eval Humming TEMPORARY"
key: lm-eval-humming-l4
@@ -1068,6 +1120,29 @@ steps:
- tests/evals/gpt_oss/
commands:
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-h100.txt
mirror:
amd:
label: ":amd: (MI300) GPQA Eval (GPT-OSS)"
dind: false
device: mi300_2
num_devices: 2
timeout_in_minutes: 40
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- csrc/
- vllm/model_executor/layers/quantization
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/v1/attention/backends/
- vllm/v1/attention/selector.py
- vllm/model_executor/layers/fused_moe/
- tests/evals/gpt_oss/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx942.txt
- label: ":nvidia: (B200) GPQA Eval (GPT-OSS)"
key: gpqa-eval-gpt-oss-2xb200
@@ -1176,6 +1251,28 @@ steps:
- tests/evals/gpt_oss/
commands:
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-spark.txt
mirror:
amd:
label: ":amd: (MI355 DPX) GPQA Eval (GPT-OSS TP1)"
# The Spark config is a backend-neutral GPT-OSS TP1 baseline.
dind: false
device: mi355_dpx
num_devices: 1
timeout_in_minutes: 40
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- csrc/
- vllm/model_executor/layers/quantization
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/v1/attention/backends/
- vllm/v1/attention/selector.py
- vllm/model_executor/layers/fused_moe/
- tests/evals/gpt_oss/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- label: ":nvidia: (H200 MIG 35GB) LM Eval KV-Offload"
key: kv-offload-small
@@ -1292,3 +1389,53 @@ steps:
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- tests/evals/mrcr/
- label: ":amd: (MI355) LM Eval Large Models Harness"
key: amd-lm-eval-large-models-harness
dind: false
device: mi355_8
optional: true
timeout_in_minutes: 85
depends_on:
- image-build-amd
working_dir: /vllm-workspace/.buildkite/lm-eval-harness
source_file_dependencies:
- csrc/
- vllm/model_executor/layers/quantization
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/v1/attention/backends/
- vllm/v1/attention/selector.py
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- .buildkite/lm-eval-harness/
num_devices: 8
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
# Qwen2-57B has 28 attention heads and cannot run at TP8.
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4
- label: ":amd: (MI300) LM Eval Large Models ROCm Harness"
key: amd-lm-eval-large-models-rocm-harness
dind: false
device: mi300_8
optional: true
timeout_in_minutes: 40
depends_on:
- image-build-amd
working_dir: /vllm-workspace/.buildkite/lm-eval-harness
source_file_dependencies:
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/model_executor/layers/quantization/
- vllm/v1/attention/backends/
- vllm/v1/attention/selector.py
- vllm/model_executor/layers/layernorm.py
- csrc/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- .buildkite/lm-eval-harness/
num_devices: 8
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm.txt --tp-size=8 --timeout=5400
+19
View File
@@ -259,6 +259,25 @@ steps:
- tests/compile
commands:
- pytest -s -v compile/fullgraph/test_full_cudagraph.py
mirror:
amd:
label: ':amd: (MI250) PyTorch Fullgraph CUDAGraph Compatibility'
dind: false
device: mi250_1
timeout_in_minutes: 65
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/compilation/
- vllm/model_executor/
- vllm/v1/attention/
- vllm/config/compilation.py
- csrc/
- tests/compile
- vllm/platforms/rocm.py
commands:
- pytest -s -v compile/fullgraph/test_full_cudagraph.py
- label: ":nvidia: (H200 MIG 18GB) PyTorch Nightly Dependency Override Check"
key: pytorch-nightly-dependency-override-check
+64
View File
@@ -97,6 +97,27 @@ steps:
- tests/v1/e2e/spec_decode/
commands:
- pytest -v -s v1/e2e/spec_decode/eagle/
mirror:
amd:
label: ":amd: (MI355) Spec Decode Eagle Nightly"
dind: false
device: mi355_1
num_devices: 1
timeout_in_minutes: 120
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/v1/sample/
- vllm/model_executor/layers/
- vllm/v1/attention/
- tests/v1/e2e/spec_decode/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
# MTP correctness is one parametrized test split into per-model folders;
# each job below runs whole directories.
@@ -234,6 +255,28 @@ steps:
commands:
- pytest -v -s v1/e2e/spec_decode/speculators/
- pytest -v -s v1/e2e/spec_decode/mtp/
mirror:
amd:
label: ":amd: (MI355) Spec Decode Speculators + MTP Nightly"
dind: false
device: mi355_1
num_devices: 1
timeout_in_minutes: 90
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/v1/sample/
- vllm/model_executor/layers/
- vllm/v1/attention/
- vllm/transformers_utils/configs/speculators/
- tests/v1/e2e/spec_decode/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- label: ":nvidia: (H200 MIG 35GB) Spec Decode N-Gram + Suffix"
key: spec-decode-ngram-suffix
@@ -304,6 +347,27 @@ steps:
- tests/v1/e2e/spec_decode/
commands:
- pytest -v -s v1/e2e/spec_decode/draft_model/
mirror:
amd:
label: ":amd: (MI355) Spec Decode Draft Model Nightly"
dind: false
device: mi355_1
num_devices: 1
timeout_in_minutes: 120
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/v1/sample/
- vllm/model_executor/layers/
- vllm/v1/attention/
- tests/v1/e2e/spec_decode/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- label: ":nvidia: (H100) Speculators Correctness Nightly"
key: speculators-correctness