[ROCm][CI] Stage E gating (#50920)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Andreas Karatzas
2026-08-28 23:52:23 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent d3d79ffc1e
commit cacc429f62
10 changed files with 234 additions and 0 deletions
+43
View File
@@ -83,6 +83,27 @@ steps:
- pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "inductor_partition and not +rms_norm and not +quant_fp8"
# Qwen/Deepseek requires +quant_fp8 as -quant_fp8 rms+quant fusion is not supported
- pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "inductor_partition and not +rms_norm and +quant_fp8 and (qwen3 or deepseek)"
mirror:
amd:
label: ":amd: (MI355) Fusion E2E Quick"
dind: false
device: mi355_1
working_dir: "/vllm-workspace/"
timeout_in_minutes: 25
depends_on:
- image-build-amd
source_file_dependencies:
- csrc/quantization/
- vllm/model_executor/
- vllm/v1/attention/
- vllm/compilation/
- tests/compile/fusions_e2e/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- rocm-smi
- "pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k 'inductor_partition and not +rms_norm and not +quant_fp8'"
- "pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k 'inductor_partition and +rms_norm and +quant_fp8 and qwen3'"
- label: ":nvidia: (H100) Fusion E2E Config Sweep"
key: fusion-e2e-config-sweep-h100
@@ -103,6 +124,28 @@ steps:
- nvidia-smi
# Run just llama3 (fp8) for all config combinations
- pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "llama-3"
mirror:
amd:
label: ":amd: (MI355) Fusion E2E Config Sweep"
dind: false
device: mi355_1
working_dir: "/vllm-workspace/"
timeout_in_minutes: 30
depends_on:
- image-build-amd
source_file_dependencies:
- csrc/quantization/
- vllm/compilation/
- vllm/model_executor/layers/layernorm.py
- vllm/model_executor/layers/activation.py
- vllm/model_executor/layers/attention/attention.py
- vllm/model_executor/layers/quantization/input_quant_fp8.py
- tests/compile/fusions_e2e/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- rocm-smi
- pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "llama-3"
- label: ":nvidia: (B200) Fusion E2E Config Sweep"
key: fusion-e2e-config-sweep-b200
+25
View File
@@ -17,6 +17,18 @@ steps:
- pytest -v -s cuda/test_cuda_context.py
- pytest -v -s cuda/test_platform_no_cuda_init.py
- pytest -v -s cuda/test_cuda_compatibility_path.py
mirror:
amd:
label: ":amd: (MI355) CUDA Platform"
dind: false
device: mi355_1
working_dir: "/vllm-workspace/tests"
timeout_in_minutes: 20
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/
- tests/cuda
- label: ":nvidia: (H200) CUDAGraph"
device: h200_35gb
@@ -34,3 +46,16 @@ steps:
- pytest -v -s v1/cudagraph/test_cudagraph_mode.py
- pytest -v -s v1/cudagraph/test_breakable_cudagraph.py
- pytest -v -s v1/cudagraph/test_encoder_cudagraph.py
mirror:
amd:
label: ":amd: (MI250) CUDAGraph"
device: mi250_1
timeout_in_minutes: 45
depends_on:
- image-build-amd
source_file_dependencies:
- tests/v1/cudagraph
- vllm/v1/cudagraph_dispatcher.py
- vllm/config/compilation.py
- vllm/compilation
- vllm/platforms/rocm.py
+16
View File
@@ -14,3 +14,19 @@ steps:
- tests/tools/test_docker_build_metadata_args.py
commands:
- pytest -v -s tools/test_docker_build_metadata_args.py
mirror:
amd:
label: ":amd: (MI250) Docker Build Metadata"
device: mi250_1
no_gpu: true
timeout_in_minutes: 35
depends_on:
- image-build-amd
source_file_dependencies:
- .buildkite/scripts/docker-build-metadata-args.sh
- .buildkite/scripts/ci-bake-rocm.sh
- docker/Dockerfile.rocm
- docker/Dockerfile.rocm_base
- docker/ci-rocm.hcl
- docker/docker-bake-rocm.hcl
- tests/tools/test_docker_build_metadata_args.py
@@ -51,3 +51,23 @@ steps:
working_dir: "/vllm-workspace"
commands:
- bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_prefetch_offload.sh 0.25 200 8030
mirror:
amd:
label: ":amd: (MI355) DeepSeek V2-Lite Prefetch Offload Accuracy"
dind: false
device: mi355_1
working_dir: "/vllm-workspace"
timeout_in_minutes: 20
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/model_executor/layers/fused_moe/
- vllm/model_executor/layers/quantization/
- vllm/v1/attention/backends/
- vllm/v1/attention/backends/mla/
- vllm/v1/attention/selector.py
- .buildkite/scripts/scheduled_integration_test/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
+14
View File
@@ -235,6 +235,13 @@ steps:
- vllm/model_executor/layers/mamba/ops
commands:
- pytest -v -s kernels/mamba
mirror:
amd:
label: ":amd: (MI250) Mamba Kernels"
device: mi250_1
timeout_in_minutes: 55
depends_on:
- image-build-amd
- label: ":nvidia: (H100) DeepGEMM Kernels"
key: kernels-deepgemm-test-h100
@@ -393,6 +400,13 @@ steps:
- pip install helion==1.4.0
- pytest -v -s kernels/helion/ --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
parallelism: 2
mirror:
amd:
label: ":amd: (MI250) Helion Kernels Shard %N"
device: mi250_1
timeout_in_minutes: 45
depends_on:
- image-build-amd
- label: ":nvidia: (H100) FP8 MoE Kernels"
+16
View File
@@ -395,3 +395,19 @@ steps:
- tests/evals/mrcr/
commands:
- pytest -s -v evals/mrcr/test_mrcr_correctness.py --config-list-file=evals/mrcr/configs/models-small.txt
mirror:
amd:
label: ":amd: (MI355) MRCR Eval Small Models"
dind: false
device: mi355_1
timeout_in_minutes: 40
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/v1/attention/backends/
- vllm/v1/attention/selector.py
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- tests/evals/mrcr/
@@ -149,3 +149,19 @@ steps:
- >-
pytest -v -s
'v1/e2e/spec_decode/mtp/test_mtp.py::test_mtp_correctness[qwen3_5-hybrid]'
mirror:
amd:
label: ":amd: (MI355) Model Runner V2 Spec Decode"
dind: false
device: mi355_1
timeout_in_minutes: 75
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/platforms/rocm.py
- vllm/v1/worker/gpu/
- vllm/v1/worker/gpu_worker.py
- tests/v1/spec_decode/test_max_len.py
- tests/v1/spec_decode/test_rejection_sampler_utils.py
- tests/v1/spec_decode/test_synthetic_rejection_sampler_utils.py
- tests/v1/e2e/spec_decode/
+8
View File
@@ -14,6 +14,14 @@ steps:
commands:
# Run a subset of model initialization tests
- pytest -v -s models/test_initialization.py::test_can_initialize_small_subset
mirror:
amd:
label: ":amd: (MI355) Basic Models (Initialization)"
dind: false
device: mi355_1
timeout_in_minutes: 25
depends_on:
- image-build-amd
- label: ":nvidia: (H200) Basic Models (Extra Initialization) Shard %N"
device: h200_35gb
@@ -144,6 +144,32 @@ steps:
- tests/models/language/generation_ppl_test
commands:
- pytest -v -s models/language/generation_ppl_test
mirror:
amd:
label: ":amd: (MI250) Language Models (PPL)"
device: mi250_1
timeout_in_minutes: 65
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/model_executor/models/qwen3_5.py
- vllm/model_executor/models/qwen3_5_mtp.py
- vllm/transformers_utils/configs/qwen3_5.py
- vllm/transformers_utils/configs/qwen3_5_moe.py
- vllm/model_executor/models/qwen2.py
- vllm/model_executor/models/qwen3.py
- vllm/model_executor/models/qwen3_next.py
- vllm/model_executor/models/qwen3_next_mtp.py
- vllm/model_executor/layers/fla/ops/
- vllm/_aiter_ops.py
- vllm/v1/attention/backends/triton_attn.py
- vllm/v1/attention/backends/rocm_attn.py
- vllm/v1/attention/backends/rocm_aiter_unified_attn.py
- vllm/v1/attention/backends/rocm_aiter_fa.py
- vllm/v1/attention/backends/flex_attention.py
- vllm/v1/attention/ops/
- vllm/platforms/rocm.py
- tests/models/language/generation_ppl_test
- label: ":nvidia: (H200) Language Models (Extended Pooling) Shard %N"
device: h200_35gb
@@ -180,3 +206,10 @@ steps:
- tests/models/language/pooling_mteb_test
commands:
- pytest -v -s models/language/pooling_mteb_test
mirror:
amd:
label: ":amd: (MI250) Language Models (MTEB)"
device: mi250_1
timeout_in_minutes: 60
depends_on:
- image-build-amd
+43
View File
@@ -36,6 +36,29 @@ steps:
- pytest -v -s entrypoints/openai/test_return_token_ids.py -k "not test_comparison"
- pytest -v -s entrypoints/serve/instrumentator/test_uds.py
- pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server"
mirror:
amd:
label: ":amd: (MI355) Rust Frontend OpenAI Coverage"
dind: false
device: mi355_1
timeout_in_minutes: 35
depends_on:
- image-build-amd
source_file_dependencies:
- rust/
- vllm/benchmarks/
- vllm/entrypoints/openai/
- vllm/entrypoints/serve/
- vllm/v1/sample/
- tests/utils.py
- tests/benchmarks/test_serve_cli.py
- tests/entrypoints/openai/chat_completion/test_chat_completion.py
- tests/entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py
- tests/entrypoints/openai/completion/test_shutdown.py
- tests/entrypoints/openai/test_return_token_ids.py
- tests/entrypoints/openai/test_uds.py
- tests/v1/sample/test_logprobs_e2e.py
- vllm/platforms/rocm.py
- label: ":nvidia: (H200) Rust Frontend Serve/Admin Coverage"
timeout_in_minutes: 25
@@ -65,6 +88,26 @@ steps:
# - pytest -v -s entrypoints/serve/dev/test_sleep.py
# /tokenizer_info is not implemented in the Rust frontend (the CLI flag is accepted as a no-op).
- pytest -v -s entrypoints/serve/tokenize/test_tokenization.py -k "not tokenizer_info"
mirror:
amd:
label: ":amd: (MI355) Rust Frontend Serve/Admin Coverage"
dind: false
device: mi355_1
timeout_in_minutes: 30
depends_on:
- image-build-amd
source_file_dependencies:
- rust/
- vllm/entrypoints/openai/
- vllm/entrypoints/serve/
- vllm/v1/engine/
- tests/utils.py
- tests/entrypoints/serve/dev/rpc/test_collective_rpc.py
- tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py
- tests/entrypoints/serve/instrumentator/test_basic.py
- tests/entrypoints/serve/instrumentator/test_metrics.py
- tests/entrypoints/serve/tokenize/test_tokenization.py
- vllm/platforms/rocm.py
- label: ":nvidia: (H200) Rust Frontend Core Correctness"
timeout_in_minutes: 20