From 8f4a7f45c53ab52b17023d3ca804e477daa36a23 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Tue, 18 Aug 2026 17:45:47 -0500 Subject: [PATCH] [ROCm][CI] Gating more ROCm tests (#44969) Signed-off-by: Andreas Karatzas Signed-off-by: Andreas Karatzas Co-authored-by: OpenAI Codex --- .buildkite/test_areas/engine.yaml | 10 ++++++- .buildkite/test_areas/entrypoints.yaml | 24 +++++++++++++++++ .buildkite/test_areas/lora.yaml | 22 +++++++++++++++ .buildkite/test_areas/misc.yaml | 8 ++++++ .buildkite/test_areas/model_runner_v2.yaml | 21 +++++++++++++++ .buildkite/test_areas/models_distributed.yaml | 27 +++++++++++++++++++ .buildkite/test_areas/models_language.yaml | 12 +++++++++ .buildkite/test_areas/models_multimodal.yaml | 24 +++++++++++++++++ .buildkite/test_areas/plugins.yaml | 11 ++++++++ .buildkite/test_areas/pytorch.yaml | 11 ++++++++ .buildkite/test_areas/spec_decode.yaml | 8 ++++++ 11 files changed, 177 insertions(+), 1 deletion(-) diff --git a/.buildkite/test_areas/engine.yaml b/.buildkite/test_areas/engine.yaml index d7bb76c69e..7cf03c7816 100644 --- a/.buildkite/test_areas/engine.yaml +++ b/.buildkite/test_areas/engine.yaml @@ -133,7 +133,7 @@ steps: depends_on: - image-build-amd -- label: ":nvidia: (H100) V1 E2E" +- label: ":nvidia: (H100) V1 E2E Hybrid Chunked Prefill" key: v1-e2e-4xh100 timeout_in_minutes: 35 device: h100 @@ -145,3 +145,11 @@ steps: - tests/v1/e2e/test_hybrid_chunked_prefill.py commands: - pytest -v -s v1/e2e/test_hybrid_chunked_prefill.py + mirror: + amd: + label: ":amd: (MI300) V1 E2E Hybrid Chunked Prefill" + dind: false + device: mi300_4 + timeout_in_minutes: 35 + depends_on: + - image-build-amd diff --git a/.buildkite/test_areas/entrypoints.yaml b/.buildkite/test_areas/entrypoints.yaml index ed143b8812..56d4e01258 100644 --- a/.buildkite/test_areas/entrypoints.yaml +++ b/.buildkite/test_areas/entrypoints.yaml @@ -14,6 +14,14 @@ steps: commands: - pytest -v -s entrypoints/unit_tests - pytest -v -s entrypoints/weight_transfer + mirror: + amd: + label: ":amd: (MI355) Entrypoints Unit" + dind: false + device: mi355_1 + timeout_in_minutes: 35 + depends_on: + - image-build-amd - label: ":nvidia: (H200) Entrypoints Integration (LLM)" device: h200_35gb @@ -144,6 +152,14 @@ steps: - tests/entrypoints/openai/responses commands: - pytest -v -s entrypoints/openai/responses + mirror: + amd: + label: ":amd: (MI355) Entrypoints Integration (Responses API)" + dind: false + device: mi355_1 + timeout_in_minutes: 50 + depends_on: + - image-build-amd - label: ":nvidia: (H200) Entrypoints Integration (Speech to Text)" device: h200_35gb @@ -170,6 +186,14 @@ steps: commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/multimodal + mirror: + amd: + label: ":amd: (MI355) Entrypoints Integration (Multimodal)" + dind: false + device: mi355_1 + timeout_in_minutes: 55 + depends_on: + - image-build-amd - label: ":nvidia: (H200) Entrypoints Integration (Pooling)" device: h200_35gb diff --git a/.buildkite/test_areas/lora.yaml b/.buildkite/test_areas/lora.yaml index 5c3c7c0e89..55a7f54d82 100644 --- a/.buildkite/test_areas/lora.yaml +++ b/.buildkite/test_areas/lora.yaml @@ -50,3 +50,25 @@ steps: - pytest -v -s -x lora/test_gptoss_tp.py - pytest -v -s -x lora/test_qwen35_densemodel_lora.py - pytest -v -s -x lora/test_gemma4_tp.py + mirror: + amd: + label: ":amd: (MI300) LoRA TP (Distributed)" + dind: false + device: mi300_4 + working_dir: "/vllm-workspace/tests" + timeout_in_minutes: 55 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/lora + - vllm/model_executor/layers/fused_moe/ + - vllm/platforms/rocm.py + - tests/lora + commands: + - pytest -v -s -x lora/test_chatglm3_tp.py + - pytest -v -s -x lora/test_llama_tp.py + - pytest -v -s -x lora/test_qwen3_with_multi_loras.py + - pytest -v -s -x lora/test_olmoe_tp.py + - pytest -v -s -x lora/test_gptoss_tp.py + - pytest -v -s -x lora/test_qwen35_densemodel_lora.py + - pytest -v -s -x lora/test_gemma4_tp.py diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index 750f51283a..a73ca9bb81 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -317,6 +317,14 @@ steps: - pytest -v -s detokenizer - pytest -v -s -m 'not cpu_test' multimodal - pytest -v -s utils_ + mirror: + amd: + label: ":amd: (MI355) Async Engine, Inputs, Utils, Worker" + dind: false + device: mi355_1 + timeout_in_minutes: 40 + depends_on: + - image-build-amd - label: ":computer: (CPU) Async Engine, Inputs, Utils, Worker, Config" key: async-engine-inputs-utils-worker-config-cpu diff --git a/.buildkite/test_areas/model_runner_v2.yaml b/.buildkite/test_areas/model_runner_v2.yaml index 1b1bef43aa..eaec866f58 100644 --- a/.buildkite/test_areas/model_runner_v2.yaml +++ b/.buildkite/test_areas/model_runner_v2.yaml @@ -81,6 +81,27 @@ steps: - export NCCL_CUMEM_HOST_ENABLE=0 - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py -k "not ray" - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py + mirror: + amd: + label: ":amd: (MI300) Model Runner V2 Distributed" + dind: false + device: mi300_2 + timeout_in_minutes: 45 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/v1/worker/gpu/ + - vllm/v1/worker/gpu_worker.py + - tests/basic_correctness/test_basic_correctness.py + - tests/v1/distributed/test_async_llm_dp.py + - tests/v1/distributed/test_eagle_dp.py + - vllm/platforms/rocm.py + commands: + - set -x + - export VLLM_USE_V2_MODEL_RUNNER=1 + - TARGET_TEST_SUITE=MI300 pytest -v -s basic_correctness/test_basic_correctness.py -m 'distributed(num_gpus=2)' -k "not ray and not True" + - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py -k "not ray" + - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py - label: ":nvidia: (L4) Model Runner V2 Pipeline Parallelism" key: model-runner-v2-pipeline-parallelism-4-gpus diff --git a/.buildkite/test_areas/models_distributed.yaml b/.buildkite/test_areas/models_distributed.yaml index 730c51b89b..e34e539ca2 100644 --- a/.buildkite/test_areas/models_distributed.yaml +++ b/.buildkite/test_areas/models_distributed.yaml @@ -22,3 +22,30 @@ steps: - pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)' - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)' + mirror: + amd: + label: ":amd: (MI300) Distributed Models" + dind: false + device: mi300_2 + timeout_in_minutes: 75 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/model_executor/model_loader/sharded_state_loader.py + - vllm/model_executor/models/ + - vllm/model_executor/layers/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - tests/basic_correctness/ + - tests/model_executor/model_loader/test_sharded_state_loader.py + - tests/models/ + commands: + - TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' + - HIP_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)' + - pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)' + - pytest models/language -v -s -m 'distributed(num_gpus=2)' + - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py + - pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)' + - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)' diff --git a/.buildkite/test_areas/models_language.yaml b/.buildkite/test_areas/models_language.yaml index e867514123..12c291efe5 100644 --- a/.buildkite/test_areas/models_language.yaml +++ b/.buildkite/test_areas/models_language.yaml @@ -114,6 +114,18 @@ steps: - MAMBA_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/state-spaces/mamba@v2.3.0' - CAUSAL_CONV1D_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0' - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)' + mirror: + amd: + label: ":amd: (MI355) Language Models (Extended Generation)" + dind: false + device: mi355_1 + timeout_in_minutes: 70 + depends_on: + - image-build-amd + commands: + - MAMBA_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr' + - CAUSAL_CONV1D_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0' + - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)' - label: ":nvidia: (H200) Language Models (PPL)" key: language-models-test-ppl diff --git a/.buildkite/test_areas/models_multimodal.yaml b/.buildkite/test_areas/models_multimodal.yaml index 1d97c3f5e6..62883c341e 100644 --- a/.buildkite/test_areas/models_multimodal.yaml +++ b/.buildkite/test_areas/models_multimodal.yaml @@ -110,6 +110,14 @@ steps: - tests/models/registry.py commands: - pytest -v -s models/multimodal/processing/test_tensor_schema.py + mirror: + amd: + label: ":amd: (MI355) Multimodal Processor" + dind: false + device: mi355_1 + timeout_in_minutes: 115 + depends_on: + - image-build-amd - label: ":nvidia: (H200) Multimodal Accuracy Eval (Small Models)" device: h200_35gb @@ -187,6 +195,14 @@ steps: - tests/models/multimodal/generation commands: - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + mirror: + amd: + label: ":amd: (MI355) Multimodal Models (Extended Generation 2) Shard %N" + dind: false + device: mi355_1 + timeout_in_minutes: 100 + depends_on: + - image-build-amd - label: ":nvidia: (H200) Multimodal Models (Extended Generation 3)" device: h200_35gb @@ -198,6 +214,14 @@ steps: - tests/models/multimodal/generation commands: - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model' + mirror: + amd: + label: ":amd: (MI355) Multimodal Models (Extended Generation 3)" + dind: false + device: mi355_1 + timeout_in_minutes: 90 + depends_on: + - image-build-amd - label: ":nvidia: (H200) Multimodal Models (Extended Pooling)" key: multi-modal-models-extended-pooling diff --git a/.buildkite/test_areas/plugins.yaml b/.buildkite/test_areas/plugins.yaml index 2cc4cb8e8b..19fa0e4d1e 100644 --- a/.buildkite/test_areas/plugins.yaml +++ b/.buildkite/test_areas/plugins.yaml @@ -49,6 +49,17 @@ steps: - pytest -v -s plugins_tests/test_oot_registration_online.py # it needs a clean process - pytest -v -s plugins_tests/test_oot_registration_offline.py # it needs a clean process - pytest -v -s plugins_tests/lora_resolvers # unit tests for in-tree lora resolver plugins + mirror: + amd: + label: ":amd: (MI250) Plugin Integration" + device: mi250_2 + timeout_in_minutes: 50 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/plugins/ + - tests/plugins/ + - vllm/platforms/rocm.py - label: ":nvidia: (H200) GGUF Plugin" diff --git a/.buildkite/test_areas/pytorch.yaml b/.buildkite/test_areas/pytorch.yaml index 78cebb2e59..810d335cf0 100644 --- a/.buildkite/test_areas/pytorch.yaml +++ b/.buildkite/test_areas/pytorch.yaml @@ -42,6 +42,17 @@ steps: # However, find does not normally propagate error codes, so we combine it with xargs # (using -0 for proper path handling) - "find compile/ -maxdepth 1 -name 'test_*.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'" + mirror: + amd: + label: ":amd: (MI300) PyTorch Compilation" + dind: false + device: mi300_1 + timeout_in_minutes: 90 + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/ + - vllm/platforms/rocm.py - label: ":nvidia: (H200) PyTorch Compilation H100 Cases" key: pytorch-compilation-unit-tests-h100 diff --git a/.buildkite/test_areas/spec_decode.yaml b/.buildkite/test_areas/spec_decode.yaml index 7486a467fb..0f3a45838b 100644 --- a/.buildkite/test_areas/spec_decode.yaml +++ b/.buildkite/test_areas/spec_decode.yaml @@ -223,6 +223,14 @@ steps: commands: - export VLLM_ALLOW_INSECURE_SERIALIZATION=1 - pytest -v -s v1/spec_decode/test_speculators_correctness.py -m slow_test + mirror: + amd: + label: ":amd: (MI300) Speculators Correctness Nightly" + dind: false + device: mi300_1 + timeout_in_minutes: 40 + depends_on: + - image-build-amd - label: ":nvidia: (B200) Spec Decode DeepSeek MTP Parallel Load" key: spec-decode-deepseek-mtp-parallel-load-2xb200-2xmi300