From d406f09eb8d05eaf81d768bf74212bc6d24cd11d Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Sat, 12 Sep 2026 02:40:14 -0500 Subject: [PATCH] [ROCm][CI] Stage F gating (#55252) Signed-off-by: Andreas Karatzas Signed-off-by: Andreas Karatzas Co-authored-by: OpenAI Codex --- .buildkite/test_areas/disaggregated.yaml | 35 ++++++++++++++ .buildkite/test_areas/distributed.yaml | 39 +++++++++++++++ .buildkite/test_areas/lm_eval.yaml | 42 +++++++++++++++++ .buildkite/test_areas/misc.yaml | 47 +++++++++++++++++++ .buildkite/test_areas/models_basic.yaml | 9 ++++ .buildkite/test_areas/models_multimodal.yaml | 9 ++++ .buildkite/test_areas/pytorch.yaml | 8 ++++ .buildkite/test_areas/rust_frontend.yaml | 29 ++++++++++++ .../test_areas/rust_frontend_cargo.yaml | 18 +++++++ 9 files changed, 236 insertions(+) diff --git a/.buildkite/test_areas/disaggregated.yaml b/.buildkite/test_areas/disaggregated.yaml index 0100b99993..5ab0e836eb 100644 --- a/.buildkite/test_areas/disaggregated.yaml +++ b/.buildkite/test_areas/disaggregated.yaml @@ -172,6 +172,23 @@ steps: commands: - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh - bash v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh + mirror: + amd: + label: ":amd: (MI355) Hybrid SSM NixlConnector PD prefix cache" + dind: false + device: mi355_2 + timeout_in_minutes: 25 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - vllm/v1/core/sched/ + - vllm/v1/core/kv_cache_coordinator.py + - tests/v1/kv_connector/nixl_integration/ + - vllm/platforms/rocm.py + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh - label: ":nvidia: (L4) MultiConnector (Nixl+Offloading) PD accuracy" key: multiconnector-nixl-offloading-pd-accuracy-2-gpus @@ -234,6 +251,24 @@ steps: commands: - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh - bash v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh + mirror: + amd: + label: ":amd: (MI355) MultiConnector (Nixl+Offloading) PD edge cases" + dind: false + device: mi355_2 + timeout_in_minutes: 25 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py + - vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py + - vllm/distributed/kv_transfer/kv_connector/v1/offloading/ + - tests/v1/kv_connector/nixl_integration/ + - vllm/platforms/rocm.py + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh # P TP 4 - D DPEP 4 test case for DSv4-Flash - label: ":nvidia: (H200) DSv4-Flash Disaggregated DP EP" diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml index 44e96ce632..dfe900d827 100644 --- a/.buildkite/test_areas/distributed.yaml +++ b/.buildkite/test_areas/distributed.yaml @@ -105,6 +105,26 @@ steps: - export NCCL_CUMEM_HOST_ENABLE=0 - pytest -v -s entrypoints/llm/test_collective_rpc.py - pytest -v -s ./compile/test_wrapper.py + mirror: + amd: + label: ":amd: (MI355) Distributed Compile + RPC" + dind: false + device: mi355_2 + timeout_in_minutes: 65 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/compilation/ + - vllm/distributed/ + - "!vllm/distributed/kv_transfer/" + - vllm/engine/ + - vllm/executor/ + - vllm/worker/worker_base.py + - vllm/v1/engine/ + - vllm/v1/worker/ + - tests/compile/test_wrapper.py + - tests/entrypoints/llm/test_collective_rpc.py + - vllm/platforms/rocm.py - label: ":nvidia: (L4) Distributed Torchrun + Shutdown" key: distributed-torchrun-shutdown-tests-2-gpus @@ -335,6 +355,25 @@ steps: - pytest -v -s distributed/test_pp_cudagraph.py - pytest -v -s distributed/test_pipeline_parallel.py - pytest -v -s v1/distributed/test_pp_dp_v2.py + mirror: + amd: + label: ":amd: (MI250) Pipeline + Context Parallelism" + device: mi250_4 + timeout_in_minutes: 90 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/distributed/ + - "!vllm/distributed/kv_transfer/" + - vllm/engine/ + - vllm/executor/ + - vllm/model_executor/models/ + - vllm/model_executor/layers/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - tests/distributed/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py - label: ":nvidia: (L4) RayExecutorV2" key: rayexecutorv2-4-gpus diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index dc90dcb852..1c2e3bb54f 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -79,6 +79,26 @@ steps: autorun_on_main: true commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small-tp.txt + mirror: + amd: + label: ":amd: (MI355) LM Eval Small Models Distributed" + dind: false + device: mi355_2 + timeout_in_minutes: 45 + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/ + - vllm/model_executor/layers/quantization + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - tests/evals/gsm8k/configs/models-mi3xx-fp8-and-mixed.txt + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt - label: ":nvidia: (B200) LM Eval PCP" key: lm-eval-pcp-4xb200 @@ -375,6 +395,28 @@ steps: commands: - uv pip install --system 'gpt-oss[eval]==0.0.5' - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-b200.txt + mirror: + amd: + label: ":amd: (MI355) GPQA Eval (GPT-OSS)" + dind: false + device: mi355_2 + timeout_in_minutes: 40 + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/ + - vllm/model_executor/layers/quantization + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - vllm/model_executor/layers/fused_moe/ + - tests/evals/gpt_oss/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + commands: + - uv pip install --system 'gpt-oss[eval]==0.0.5' + - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt - label: ":nvidia: (B200) Qwen3.8-Flash-Next-FP8 Accuracy Eval" key: accuracy-eval-qwen3-8-flash-next-fp8-b200 diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index 5ae3bcf73e..d24ec5b57f 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -251,6 +251,35 @@ steps: commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s -m 'distributed' tests/v1/kv_connector/extract_hidden_states_integration + mirror: + amd: + label: ":amd: (MI355) Extract Hidden States Integration" + dind: false + device: mi355_2 + timeout_in_minutes: 20 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/config/speculative.py + - vllm/distributed/kv_transfer/kv_connector/ + - vllm/model_executor/layers/attention/ + - vllm/model_executor/layers/mamba/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/models/extract_hidden_states.py + - vllm/model_executor/models/llama.py + - vllm/model_executor/models/qwen3_5.py + - vllm/model_executor/models/qwen3_next.py + - vllm/model_executor/models/registry.py + - vllm/transformers_utils/configs/extract_hidden_states.py + - vllm/transformers_utils/configs/qwen3_5.py + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - vllm/v1/kv_cache_interface.py + - vllm/v1/spec_decode/extract_hidden_states.py + - vllm/v1/worker/gpu_model_runner.py + - vllm/_aiter_ops.py + - tests/v1/kv_connector/extract_hidden_states_integration/ + - vllm/platforms/rocm.py - label: ":nvidia: (H200 MIG 18GB) Regression" key: regression @@ -537,6 +566,24 @@ steps: - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA - VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k FLASH_ATTN + mirror: + amd: + label: ":amd: (MI250) Batch Invariance" + device: mi250_1 + timeout_in_minutes: 30 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/v1/attention + - vllm/model_executor/layers + - tests/v1/determinism/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pip install pytest-timeout pytest-forked + - pytest -v -s v1/determinism/test_batch_invariance.py + - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py - label: ":nvidia: (B200) Batch Invariance" key: batch-invariance-b200 diff --git a/.buildkite/test_areas/models_basic.yaml b/.buildkite/test_areas/models_basic.yaml index 24447cb501..bf453a12d8 100644 --- a/.buildkite/test_areas/models_basic.yaml +++ b/.buildkite/test_areas/models_basic.yaml @@ -156,3 +156,12 @@ steps: device: cpu-small commands: - pytest -v -s models/test_utils.py models/test_vision.py models/test_adapters.py models/test_qwen3_5_mtp_config.py models/transformers/fusers/ + mirror: + amd: + label: ":amd: (MI250) Basic Models Other" + device: mi250_1 + no_gpu: true + working_dir: "/vllm-workspace/tests" + timeout_in_minutes: 25 + depends_on: + - image-build-amd diff --git a/.buildkite/test_areas/models_multimodal.yaml b/.buildkite/test_areas/models_multimodal.yaml index 9893383bff..b09e8cc2d9 100644 --- a/.buildkite/test_areas/models_multimodal.yaml +++ b/.buildkite/test_areas/models_multimodal.yaml @@ -98,6 +98,15 @@ steps: commands: - pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB parallelism: 4 + mirror: + amd: + label: ":amd: (MI250) Multimodal Processor Shard %N" + device: mi250_1 + no_gpu: true + working_dir: "/vllm-workspace/tests" + timeout_in_minutes: 80 + depends_on: + - image-build-amd - label: ":nvidia: (H200 MIG 18GB) Multimodal Processor" key: multi-modal-processor diff --git a/.buildkite/test_areas/pytorch.yaml b/.buildkite/test_areas/pytorch.yaml index eb5b840703..f34ff50e67 100644 --- a/.buildkite/test_areas/pytorch.yaml +++ b/.buildkite/test_areas/pytorch.yaml @@ -160,6 +160,14 @@ steps: # Use `find` to launch multiple instances of pytest so that # they do not suffer from https://github.com/vllm-project/vllm/issues/28965 - "find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_cudagraph.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'" + mirror: + amd: + label: ":amd: (MI355) PyTorch Fullgraph" + dind: false + device: mi355_1 + timeout_in_minutes: 90 + depends_on: + - image-build-amd # Hopper-only DeepSeek-V2-Lite cases in this file require two 29.3-GiB model # instances and cannot fit a 35GB MIG slice. L4 retains the original coverage: diff --git a/.buildkite/test_areas/rust_frontend.yaml b/.buildkite/test_areas/rust_frontend.yaml index 97078512a1..770d6b64df 100644 --- a/.buildkite/test_areas/rust_frontend.yaml +++ b/.buildkite/test_areas/rust_frontend.yaml @@ -131,6 +131,20 @@ steps: - export VLLM_USE_RUST_FRONTEND=1 - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine + mirror: + amd: + label: ":amd: (MI355) Rust Frontend Core Correctness" + dind: false + device: mi355_1 + timeout_in_minutes: 20 + depends_on: + - image-build-amd + source_file_dependencies: + - rust/ + - vllm/entrypoints/openai/ + - tests/utils.py + - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/platforms/rocm.py - label: ":nvidia: (H200 MIG 35GB) Rust Frontend Tool Use" key: rust-frontend-tool-use @@ -147,6 +161,21 @@ steps: - export VLLM_USE_RUST_FRONTEND=1 - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s tool_use --ignore=tool_use/mistral --models llama3.2 -k "not test_response_format_with_tool_choice_required and not test_parallel_tool_calls_false and not test_tool_call_and_choice" + mirror: + amd: + label: ":amd: (MI355) Rust Frontend Tool Use" + dind: false + device: mi355_1 + timeout_in_minutes: 25 + depends_on: + - image-build-amd + source_file_dependencies: + - rust/ + - vllm/entrypoints/openai/ + - vllm/tool_parsers/ + - tests/utils.py + - tests/tool_use/ + - vllm/platforms/rocm.py - label: ":nvidia: (L4) Rust Frontend Distributed" key: rust-frontend-distributed diff --git a/.buildkite/test_areas/rust_frontend_cargo.yaml b/.buildkite/test_areas/rust_frontend_cargo.yaml index 11bca62526..0b49787d5d 100644 --- a/.buildkite/test_areas/rust_frontend_cargo.yaml +++ b/.buildkite/test_areas/rust_frontend_cargo.yaml @@ -14,6 +14,15 @@ steps: - .buildkite/scripts/run-rust-frontend-cargo-ci.sh commands: - .buildkite/scripts/run-rust-frontend-cargo-ci.sh style-clippy + mirror: + amd: + label: ":amd: (MI250) Rust Frontend Cargo Style + Clippy" + device: mi250_1 + no_gpu: true + working_dir: "/vllm-workspace" + timeout_in_minutes: 35 + depends_on: + - image-build-amd - label: ":computer: (CPU) Rust Frontend Cargo" key: rust-frontend-cargo-tests @@ -28,3 +37,12 @@ steps: - .buildkite/scripts/run-rust-frontend-cargo-ci.sh commands: - .buildkite/scripts/run-rust-frontend-cargo-ci.sh test + mirror: + amd: + label: ":amd: (MI250) Rust Frontend Cargo" + device: mi250_1 + no_gpu: true + working_dir: "/vllm-workspace" + timeout_in_minutes: 35 + depends_on: + - image-build-amd