From cacc429f62c3738c9c95093e9bd410e96103221a Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Fri, 28 Aug 2026 23:52:23 -0700 Subject: [PATCH] [ROCm][CI] Stage E gating (#50920) Signed-off-by: Andreas Karatzas Signed-off-by: Andreas Karatzas Co-authored-by: OpenAI Codex --- .buildkite/test_areas/compile.yaml | 43 ++++++++++++++++++++++ .buildkite/test_areas/cuda.yaml | 25 +++++++++++++ .buildkite/test_areas/docker.yaml | 16 ++++++++ .buildkite/test_areas/e2e_integration.yaml | 20 ++++++++++ .buildkite/test_areas/kernels.yaml | 14 +++++++ .buildkite/test_areas/lm_eval.yaml | 16 ++++++++ .buildkite/test_areas/model_runner_v2.yaml | 16 ++++++++ .buildkite/test_areas/models_basic.yaml | 8 ++++ .buildkite/test_areas/models_language.yaml | 33 +++++++++++++++++ .buildkite/test_areas/rust_frontend.yaml | 43 ++++++++++++++++++++++ 10 files changed, 234 insertions(+) diff --git a/.buildkite/test_areas/compile.yaml b/.buildkite/test_areas/compile.yaml index dc847bf2c3..ac0112ad5e 100644 --- a/.buildkite/test_areas/compile.yaml +++ b/.buildkite/test_areas/compile.yaml @@ -83,6 +83,27 @@ steps: - pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "inductor_partition and not +rms_norm and not +quant_fp8" # Qwen/Deepseek requires +quant_fp8 as -quant_fp8 rms+quant fusion is not supported - pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "inductor_partition and not +rms_norm and +quant_fp8 and (qwen3 or deepseek)" + mirror: + amd: + label: ":amd: (MI355) Fusion E2E Quick" + dind: false + device: mi355_1 + working_dir: "/vllm-workspace/" + timeout_in_minutes: 25 + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/quantization/ + - vllm/model_executor/ + - vllm/v1/attention/ + - vllm/compilation/ + - tests/compile/fusions_e2e/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + commands: + - rocm-smi + - "pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k 'inductor_partition and not +rms_norm and not +quant_fp8'" + - "pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k 'inductor_partition and +rms_norm and +quant_fp8 and qwen3'" - label: ":nvidia: (H100) Fusion E2E Config Sweep" key: fusion-e2e-config-sweep-h100 @@ -103,6 +124,28 @@ steps: - nvidia-smi # Run just llama3 (fp8) for all config combinations - pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "llama-3" + mirror: + amd: + label: ":amd: (MI355) Fusion E2E Config Sweep" + dind: false + device: mi355_1 + working_dir: "/vllm-workspace/" + timeout_in_minutes: 30 + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/quantization/ + - vllm/compilation/ + - vllm/model_executor/layers/layernorm.py + - vllm/model_executor/layers/activation.py + - vllm/model_executor/layers/attention/attention.py + - vllm/model_executor/layers/quantization/input_quant_fp8.py + - tests/compile/fusions_e2e/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + commands: + - rocm-smi + - pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "llama-3" - label: ":nvidia: (B200) Fusion E2E Config Sweep" key: fusion-e2e-config-sweep-b200 diff --git a/.buildkite/test_areas/cuda.yaml b/.buildkite/test_areas/cuda.yaml index 7ddbaeec65..a491eed274 100644 --- a/.buildkite/test_areas/cuda.yaml +++ b/.buildkite/test_areas/cuda.yaml @@ -17,6 +17,18 @@ steps: - pytest -v -s cuda/test_cuda_context.py - pytest -v -s cuda/test_platform_no_cuda_init.py - pytest -v -s cuda/test_cuda_compatibility_path.py + mirror: + amd: + label: ":amd: (MI355) CUDA Platform" + dind: false + device: mi355_1 + working_dir: "/vllm-workspace/tests" + timeout_in_minutes: 20 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/ + - tests/cuda - label: ":nvidia: (H200) CUDAGraph" device: h200_35gb @@ -34,3 +46,16 @@ steps: - pytest -v -s v1/cudagraph/test_cudagraph_mode.py - pytest -v -s v1/cudagraph/test_breakable_cudagraph.py - pytest -v -s v1/cudagraph/test_encoder_cudagraph.py + mirror: + amd: + label: ":amd: (MI250) CUDAGraph" + device: mi250_1 + timeout_in_minutes: 45 + depends_on: + - image-build-amd + source_file_dependencies: + - tests/v1/cudagraph + - vllm/v1/cudagraph_dispatcher.py + - vllm/config/compilation.py + - vllm/compilation + - vllm/platforms/rocm.py diff --git a/.buildkite/test_areas/docker.yaml b/.buildkite/test_areas/docker.yaml index 4bd36fad18..e88f19ad97 100644 --- a/.buildkite/test_areas/docker.yaml +++ b/.buildkite/test_areas/docker.yaml @@ -14,3 +14,19 @@ steps: - tests/tools/test_docker_build_metadata_args.py commands: - pytest -v -s tools/test_docker_build_metadata_args.py + mirror: + amd: + label: ":amd: (MI250) Docker Build Metadata" + device: mi250_1 + no_gpu: true + timeout_in_minutes: 35 + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/scripts/docker-build-metadata-args.sh + - .buildkite/scripts/ci-bake-rocm.sh + - docker/Dockerfile.rocm + - docker/Dockerfile.rocm_base + - docker/ci-rocm.hcl + - docker/docker-bake-rocm.hcl + - tests/tools/test_docker_build_metadata_args.py diff --git a/.buildkite/test_areas/e2e_integration.yaml b/.buildkite/test_areas/e2e_integration.yaml index 8f0e2042bf..02ba409e34 100644 --- a/.buildkite/test_areas/e2e_integration.yaml +++ b/.buildkite/test_areas/e2e_integration.yaml @@ -51,3 +51,23 @@ steps: working_dir: "/vllm-workspace" commands: - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_prefetch_offload.sh 0.25 200 8030 + mirror: + amd: + label: ":amd: (MI355) DeepSeek V2-Lite Prefetch Offload Accuracy" + dind: false + device: mi355_1 + working_dir: "/vllm-workspace" + timeout_in_minutes: 20 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/quantization/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/backends/mla/ + - vllm/v1/attention/selector.py + - .buildkite/scripts/scheduled_integration_test/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py diff --git a/.buildkite/test_areas/kernels.yaml b/.buildkite/test_areas/kernels.yaml index 6197790c03..49754ad81b 100644 --- a/.buildkite/test_areas/kernels.yaml +++ b/.buildkite/test_areas/kernels.yaml @@ -235,6 +235,13 @@ steps: - vllm/model_executor/layers/mamba/ops commands: - pytest -v -s kernels/mamba + mirror: + amd: + label: ":amd: (MI250) Mamba Kernels" + device: mi250_1 + timeout_in_minutes: 55 + depends_on: + - image-build-amd - label: ":nvidia: (H100) DeepGEMM Kernels" key: kernels-deepgemm-test-h100 @@ -393,6 +400,13 @@ steps: - pip install helion==1.4.0 - pytest -v -s kernels/helion/ --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT parallelism: 2 + mirror: + amd: + label: ":amd: (MI250) Helion Kernels Shard %N" + device: mi250_1 + timeout_in_minutes: 45 + depends_on: + - image-build-amd - label: ":nvidia: (H100) FP8 MoE Kernels" diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index 58e6717aca..bb659bb749 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -395,3 +395,19 @@ steps: - tests/evals/mrcr/ commands: - pytest -s -v evals/mrcr/test_mrcr_correctness.py --config-list-file=evals/mrcr/configs/models-small.txt + mirror: + amd: + label: ":amd: (MI355) MRCR Eval Small Models" + dind: false + device: mi355_1 + timeout_in_minutes: 40 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - tests/evals/mrcr/ diff --git a/.buildkite/test_areas/model_runner_v2.yaml b/.buildkite/test_areas/model_runner_v2.yaml index 0b47c8f545..e6d93d330b 100644 --- a/.buildkite/test_areas/model_runner_v2.yaml +++ b/.buildkite/test_areas/model_runner_v2.yaml @@ -149,3 +149,19 @@ steps: - >- pytest -v -s 'v1/e2e/spec_decode/mtp/test_mtp.py::test_mtp_correctness[qwen3_5-hybrid]' + mirror: + amd: + label: ":amd: (MI355) Model Runner V2 Spec Decode" + dind: false + device: mi355_1 + timeout_in_minutes: 75 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/platforms/rocm.py + - vllm/v1/worker/gpu/ + - vllm/v1/worker/gpu_worker.py + - tests/v1/spec_decode/test_max_len.py + - tests/v1/spec_decode/test_rejection_sampler_utils.py + - tests/v1/spec_decode/test_synthetic_rejection_sampler_utils.py + - tests/v1/e2e/spec_decode/ diff --git a/.buildkite/test_areas/models_basic.yaml b/.buildkite/test_areas/models_basic.yaml index bdb3cab5a6..f6d0915564 100644 --- a/.buildkite/test_areas/models_basic.yaml +++ b/.buildkite/test_areas/models_basic.yaml @@ -14,6 +14,14 @@ steps: commands: # Run a subset of model initialization tests - pytest -v -s models/test_initialization.py::test_can_initialize_small_subset + mirror: + amd: + label: ":amd: (MI355) Basic Models (Initialization)" + dind: false + device: mi355_1 + timeout_in_minutes: 25 + depends_on: + - image-build-amd - label: ":nvidia: (H200) Basic Models (Extra Initialization) Shard %N" device: h200_35gb diff --git a/.buildkite/test_areas/models_language.yaml b/.buildkite/test_areas/models_language.yaml index 48f5b0d374..be2e36c301 100644 --- a/.buildkite/test_areas/models_language.yaml +++ b/.buildkite/test_areas/models_language.yaml @@ -144,6 +144,32 @@ steps: - tests/models/language/generation_ppl_test commands: - pytest -v -s models/language/generation_ppl_test + mirror: + amd: + label: ":amd: (MI250) Language Models (PPL)" + device: mi250_1 + timeout_in_minutes: 65 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/model_executor/models/qwen3_5.py + - vllm/model_executor/models/qwen3_5_mtp.py + - vllm/transformers_utils/configs/qwen3_5.py + - vllm/transformers_utils/configs/qwen3_5_moe.py + - vllm/model_executor/models/qwen2.py + - vllm/model_executor/models/qwen3.py + - vllm/model_executor/models/qwen3_next.py + - vllm/model_executor/models/qwen3_next_mtp.py + - vllm/model_executor/layers/fla/ops/ + - vllm/_aiter_ops.py + - vllm/v1/attention/backends/triton_attn.py + - vllm/v1/attention/backends/rocm_attn.py + - vllm/v1/attention/backends/rocm_aiter_unified_attn.py + - vllm/v1/attention/backends/rocm_aiter_fa.py + - vllm/v1/attention/backends/flex_attention.py + - vllm/v1/attention/ops/ + - vllm/platforms/rocm.py + - tests/models/language/generation_ppl_test - label: ":nvidia: (H200) Language Models (Extended Pooling) Shard %N" device: h200_35gb @@ -180,3 +206,10 @@ steps: - tests/models/language/pooling_mteb_test commands: - pytest -v -s models/language/pooling_mteb_test + mirror: + amd: + label: ":amd: (MI250) Language Models (MTEB)" + device: mi250_1 + timeout_in_minutes: 60 + depends_on: + - image-build-amd diff --git a/.buildkite/test_areas/rust_frontend.yaml b/.buildkite/test_areas/rust_frontend.yaml index 21fcd127db..b388b8d0f7 100644 --- a/.buildkite/test_areas/rust_frontend.yaml +++ b/.buildkite/test_areas/rust_frontend.yaml @@ -36,6 +36,29 @@ steps: - pytest -v -s entrypoints/openai/test_return_token_ids.py -k "not test_comparison" - pytest -v -s entrypoints/serve/instrumentator/test_uds.py - pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server" + mirror: + amd: + label: ":amd: (MI355) Rust Frontend OpenAI Coverage" + dind: false + device: mi355_1 + timeout_in_minutes: 35 + depends_on: + - image-build-amd + source_file_dependencies: + - rust/ + - vllm/benchmarks/ + - vllm/entrypoints/openai/ + - vllm/entrypoints/serve/ + - vllm/v1/sample/ + - tests/utils.py + - tests/benchmarks/test_serve_cli.py + - tests/entrypoints/openai/chat_completion/test_chat_completion.py + - tests/entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py + - tests/entrypoints/openai/completion/test_shutdown.py + - tests/entrypoints/openai/test_return_token_ids.py + - tests/entrypoints/openai/test_uds.py + - tests/v1/sample/test_logprobs_e2e.py + - vllm/platforms/rocm.py - label: ":nvidia: (H200) Rust Frontend Serve/Admin Coverage" timeout_in_minutes: 25 @@ -65,6 +88,26 @@ steps: # - pytest -v -s entrypoints/serve/dev/test_sleep.py # /tokenizer_info is not implemented in the Rust frontend (the CLI flag is accepted as a no-op). - pytest -v -s entrypoints/serve/tokenize/test_tokenization.py -k "not tokenizer_info" + mirror: + amd: + label: ":amd: (MI355) Rust Frontend Serve/Admin Coverage" + dind: false + device: mi355_1 + timeout_in_minutes: 30 + depends_on: + - image-build-amd + source_file_dependencies: + - rust/ + - vllm/entrypoints/openai/ + - vllm/entrypoints/serve/ + - vllm/v1/engine/ + - tests/utils.py + - tests/entrypoints/serve/dev/rpc/test_collective_rpc.py + - tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py + - tests/entrypoints/serve/instrumentator/test_basic.py + - tests/entrypoints/serve/instrumentator/test_metrics.py + - tests/entrypoints/serve/tokenize/test_tokenization.py + - vllm/platforms/rocm.py - label: ":nvidia: (H200) Rust Frontend Core Correctness" timeout_in_minutes: 20