[ROCm][CI] Stage H gating and MI355 test reallocation (#57080)

Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>

.buildkite/test_areas/compile.yaml

@@ -92,7 +92,7 @@ steps:
       dind: false
       device: mi355_1
       working_dir: "/vllm-workspace/"
-      timeout_in_minutes: 25
+      timeout_in_minutes: 30
       depends_on:
       - image-build-amd
       source_file_dependencies:
@@ -133,7 +133,7 @@ steps:
       dind: false
       device: mi355_1
       working_dir: "/vllm-workspace/"
-      timeout_in_minutes: 30
+      timeout_in_minutes: 40
       depends_on:
       - image-build-amd
       source_file_dependencies:

.buildkite/test_areas/cuda.yaml

@@ -23,7 +23,7 @@ steps:
       dind: false
       device: mi355_1
       working_dir: "/vllm-workspace/tests"
-      timeout_in_minutes: 20
+      timeout_in_minutes: 25
       depends_on:
       - image-build-amd
       source_file_dependencies:
@@ -50,7 +50,7 @@ steps:
     amd:
       label: ":amd: (MI250) CUDAGraph"
       device: mi250_1
-      timeout_in_minutes: 45
+      timeout_in_minutes: 55
       depends_on:
       - image-build-amd
       source_file_dependencies:

.buildkite/test_areas/disaggregated.yaml

@@ -16,9 +16,9 @@ steps:
     - bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
   mirror:
     amd:
-      label: ":amd: (MI300) Distributed NixlConnector PD accuracy"
+      label: ":amd: (MI355) Distributed NixlConnector PD accuracy"
       dind: false
-      device: mi300_4
+      device: mi355_4
       timeout_in_minutes: 60
       depends_on:
         - image-build-amd
@@ -71,9 +71,9 @@ steps:
     - DP_EP=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
   mirror:
     amd:
-      label: ":amd: (MI300) DP EP Distributed NixlConnector PD accuracy"
+      label: ":amd: (MI355) DP EP Distributed NixlConnector PD accuracy"
       dind: false
-      device: mi300_4
+      device: mi355_4
       timeout_in_minutes: 40
       depends_on:
         - image-build-amd
@@ -130,7 +130,7 @@ steps:
       label: ":amd: (MI300) Hybrid SSM NixlConnector PD accuracy"
       dind: false
       device: mi300_4
-      timeout_in_minutes: 55
+      timeout_in_minutes: 60
       depends_on:
         - image-build-amd
       source_file_dependencies:
@@ -221,9 +221,9 @@ steps:
     - bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh
   mirror:
     amd:
-      label: ":amd: (MI300) NixlConnector PD + Spec Decode acceptance"
+      label: ":amd: (MI355) NixlConnector PD + Spec Decode acceptance"
       dind: false
-      device: mi300_2
+      device: mi355_2
       timeout_in_minutes: 45
       depends_on:
         - image-build-amd

.buildkite/test_areas/distributed.yaml

@@ -65,9 +65,9 @@ steps:
   - pytest -v -s v1/e2e/spec_decode/test_sharded_sampling.py
   mirror:
     amd:
-      label: ":amd: (MI300) Distributed DP Basic"
+      label: ":amd: (MI355) Distributed DP Basic"
       dind: false
-      device: mi300_2
+      device: mi355_2
       timeout_in_minutes: 45
       depends_on:
       - image-build-amd
@@ -125,6 +125,9 @@ steps:
       - tests/compile/test_wrapper.py
       - tests/entrypoints/llm/test_collective_rpc.py
       - vllm/platforms/rocm.py
+      commands:
+      - pytest -v -s entrypoints/llm/test_collective_rpc.py
+      - pytest -v -s ./compile/test_wrapper.py
 
 - label: ":nvidia: (L4) Distributed Torchrun + Shutdown"
   key: distributed-torchrun-shutdown-tests-2-gpus
@@ -296,6 +299,57 @@ steps:
     - pytest -v -s tests/v1/distributed/test_dbo.py
     - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py
     - pytest -v -s tests/distributed/test_packed_tensor.py
+  mirror:
+    amd:
+      label: ':amd: (MI355) Distributed Features'
+      dind: false
+      device: mi355_2
+      timeout_in_minutes: 90
+      working_dir: /vllm-workspace/
+      depends_on:
+      - image-build-amd
+      source_file_dependencies:
+      - vllm/distributed/
+      - vllm/v1/distributed/
+      - vllm/model_executor/layers/fused_moe/
+      - vllm/v1/attention/backends/
+      - vllm/v1/attention/selector.py
+      - tests/v1/distributed/test_dbo.py
+      - tests/distributed/test_context_parallel.py
+      - examples/features/data_parallel/data_parallel_offline.py
+      - vllm/_aiter_ops.py
+      - vllm/platforms/rocm.py
+      - csrc/custom_quickreduce.cu
+      - csrc/ops.h
+      - csrc/torch_bindings.cpp
+      - vllm/model_executor/layers/
+      - vllm/entrypoints/llm.py
+      - vllm/config/parallel.py
+      - vllm/v1/engine/
+      - vllm/v1/executor/
+      - vllm/v1/worker/
+      - vllm/_custom_ops.py
+      - vllm/envs.py
+      - tests/distributed/test_rocm_aiter_custom_ar.py
+      - tests/distributed/test_rocm_quick_reduce.py
+      - tests/distributed/test_quick_all_reduce.py
+      - tests/v1/e2e/general/test_rocm_aiter_custom_ar.py
+      - tests/utils.py
+      - examples/rl/rlhf_async_new_apis.py
+      - tests/distributed/test_weight_transfer.py
+      - tests/distributed/test_packed_tensor.py
+      commands:
+      - pytest -v -s tests/distributed/test_context_parallel.py
+      - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py
+      - VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput
+      - VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=allgather_reducescatter --disable-nccl-for-dp-synchronization
+      - pytest -v -s tests/v1/distributed/test_dbo.py
+      - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py
+      - pytest -v -s tests/distributed/test_packed_tensor.py
+      - pytest -v -s tests/distributed/test_rocm_aiter_custom_ar.py
+      - pytest -v -s tests/v1/e2e/general/test_rocm_aiter_custom_ar.py
+      - pytest -v -s tests/distributed/test_rocm_quick_reduce.py
+      - pytest -v -s tests/distributed/test_quick_all_reduce.py
 
 - label: ":nvidia: (B200) Distributed"
   key: distributed-tests-2xb200

.buildkite/test_areas/e2e_integration.yaml

@@ -57,7 +57,7 @@ steps:
       dind: false
       device: mi355_1
       working_dir: "/vllm-workspace"
-      timeout_in_minutes: 20
+      timeout_in_minutes: 30
       depends_on:
       - image-build-amd
       source_file_dependencies:

.buildkite/test_areas/engine.yaml

@@ -56,6 +56,8 @@ steps:
       timeout_in_minutes: 45
       depends_on:
       - image-build-amd
+      source_file_dependencies:
+      - vllm/platforms/rocm.py
 
 - label: ":nvidia: (H200 MIG 18GB) E2E Scheduling"
   key: e2e-scheduling-1-gpu
@@ -73,6 +75,8 @@ steps:
       timeout_in_minutes: 55
       depends_on:
       - image-build-amd
+      source_file_dependencies:
+      - vllm/platforms/rocm.py
 
 - label: ":nvidia: (H200 MIG 18GB) E2E Core"
   device: h200_18gb

.buildkite/test_areas/entrypoints.yaml

@@ -21,9 +21,11 @@ steps:
       label: ":amd: (MI355) Entrypoints Unit"
       dind: false
       device: mi355_1
-      timeout_in_minutes: 35
+      timeout_in_minutes: 50
       depends_on:
       - image-build-amd
+      source_file_dependencies:
+      - vllm/platforms/rocm.py
 
 - label: ":nvidia: (H200 MIG 35GB) Entrypoints Integration (LLM)"
   device: h200_35gb
@@ -44,7 +46,7 @@ steps:
       label: ":amd: (MI300) Entrypoints Integration (LLM)"
       dind: false
       device: mi300_1
-      timeout_in_minutes: 55
+      timeout_in_minutes: 90
       depends_on:
       - image-build-amd
 
@@ -176,6 +178,15 @@ steps:
   commands:
   - export VLLM_WORKER_MULTIPROC_METHOD=spawn
   - pytest -v -s entrypoints/speech_to_text
+  mirror:
+    amd:
+      label: ":amd: (MI355) Entrypoints Integration (Speech to Text)"
+      dind: false
+      device: mi355_1
+      timeout_in_minutes: 60
+      working_dir: "/vllm-workspace/tests"
+      depends_on:
+      - image-build-amd
 
 - label: ":nvidia: (H200 MIG 35GB) Entrypoints Integration (Multimodal)"
   device: h200_35gb
@@ -210,6 +221,15 @@ steps:
   commands:
   - export VLLM_WORKER_MULTIPROC_METHOD=spawn
   - pytest -v -s entrypoints/pooling
+  mirror:
+    amd:
+      label: ":amd: (MI355) Entrypoints Integration (Pooling)"
+      dind: false
+      device: mi355_1
+      timeout_in_minutes: 65
+      working_dir: "/vllm-workspace/tests"
+      depends_on:
+      - image-build-amd
 
 - label: ":nvidia: (H200 MIG 18GB) OpenAI API Correctness"
   key: openai-api-correctness

.buildkite/test_areas/kernels.yaml

@@ -13,6 +13,22 @@ steps:
   commands:
     - pytest -v -s tests/ir
     - pytest -v -s tests/kernels/ir
+  mirror:
+    amd:
+      label: ":amd: (MI355) vLLM IR"
+      dind: false
+      device: mi355_1
+      timeout_in_minutes: 50
+      working_dir: "/vllm-workspace/"
+      depends_on:
+      - image-build-amd
+      source_file_dependencies:
+      - tests/ir/
+      - tests/kernels/ir/
+      - vllm/ir
+      - vllm/kernels
+      - vllm/_aiter_ops.py
+      - vllm/platforms/rocm.py
 
 - label: ":nvidia: (H200 MIG 35GB) Core Operation Kernels Shard %N"
   device: h200_35gb
@@ -26,6 +42,29 @@ steps:
   commands:
     - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_concat_mla_q.py kernels/test_fused_qk_norm_rope_gate.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
   parallelism: 3
+  mirror:
+    amd:
+      label: ":amd: (MI355) Core Operation Kernels Shard %N"
+      dind: false
+      device: mi355_1
+      timeout_in_minutes: 40
+      working_dir: "/vllm-workspace/tests"
+      depends_on:
+      - image-build-amd
+      source_file_dependencies:
+      - csrc/
+      - tests/kernels/core
+      - tests/kernels/test_top_k_per_row.py
+      - tests/kernels/test_concat_mla_q.py
+      - tests/kernels/test_rocm_fp32_router_gemm.py
+      - vllm/model_executor/layers/fused_moe/router/gate_linear.py
+      - vllm/model_executor/layers/fused_moe/router/rocm_fp32_router_gemm.py
+      - vllm/model_executor/layers/rotary_embedding/
+      - vllm/_aiter_ops.py
+      - vllm/platforms/rocm.py
+      - tests/kernels/test_fused_qk_norm_rope_gate.py
+      commands:
+      - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_fused_qk_norm_rope_gate.py kernels/test_concat_mla_q.py kernels/test_top_k_per_row.py kernels/test_rocm_fp32_router_gemm.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
 
 - label: ":nvidia: (H100) MiniMax Reduce RMS Kernels"
   key: kernels-minimax-reduce-rms-test-2-gpus
@@ -145,6 +184,7 @@ steps:
       dind: false
       device: mi300_1
       timeout_in_minutes: 90
+      parallelism: 6
       depends_on:
       - image-build-amd
       source_file_dependencies:
@@ -208,9 +248,6 @@ steps:
       label: ":amd: (MI300) Quantization Kernels Shard %N"
       dind: false
       device: mi300_1
-      # Pin the AMD mirror at its current 2 shards so raising NVIDIA
-      # parallelism does not multiply AMD copies (needs ci-infra #473).
-      parallelism: 2
       timeout_in_minutes: 120
       source_file_dependencies:
       - csrc/quantization/
@@ -222,6 +259,12 @@ steps:
       - vllm/kernels/aiter_ops.py
       - vllm/platforms/rocm.py
       - vllm/model_executor/kernels/
+      - csrc/rocm/
+      - vllm/v1/attention/backends/rocm_aiter_fa.py
+      - tests/kernels/quant_utils.py
+      - tests/kernels/utils.py
+      - vllm/_custom_ops.py
+      - vllm/envs.py
       depends_on:
       - image-build-amd
 
@@ -258,6 +301,10 @@ steps:
       - vllm/config
       - vllm/_aiter_ops.py
       - vllm/platforms/rocm.py
+      - csrc/rocm/
+      - tests/kernels/utils.py
+      - vllm/model_executor/layers/quantization/mxfp4.py
+      - vllm/model_executor/layers/quantization/utils/quant_utils.py
       depends_on:
       - image-build-amd
 
@@ -278,6 +325,8 @@ steps:
       timeout_in_minutes: 55
       depends_on:
       - image-build-amd
+      source_file_dependencies:
+      - vllm/platforms/rocm.py
 
 - label: ":nvidia: (H100) DeepGEMM Kernels"
   key: kernels-deepgemm-test-h100
@@ -463,6 +512,9 @@ steps:
       timeout_in_minutes: 45
       depends_on:
       - image-build-amd
+      source_file_dependencies:
+      - vllm/platforms/rocm.py
+      - scripts/benchmark_helion_kernels.py
 
  
 - label: ":nvidia: (H100) FP8 MoE Kernels"
@@ -491,6 +543,25 @@ steps:
   commands:
     - pytest -v -s kernels/moe/test_deepep_deepgemm_moe.py
     - pytest -v -s kernels/moe/test_deepep_moe.py
+  mirror:
+    amd:
+      label: ":amd: (MI355) DeepEP FP8 MoE Kernels"
+      dind: false
+      device: mi355_2
+      timeout_in_minutes: 55
+      working_dir: "/vllm-workspace/tests"
+      depends_on:
+      - image-build-amd
+      source_file_dependencies:
+      - csrc/moe/
+      - csrc/quantization/w8a8/cutlass/moe/
+      - vllm/model_executor/layers/fused_moe/
+      - tests/kernels/moe/test_deepep_moe.py
+      - vllm/_aiter_ops.py
+      - vllm/platforms/rocm.py
+      - vllm/envs.py
+      commands:
+      - pytest -v -s kernels/moe/test_deepep_moe.py
 
 - label: ":nvidia: (B200) FP4 MoE Kernels"
   key: kernels-fp4-moe-test-b200
@@ -520,7 +591,30 @@ steps:
   - vllm/config
   commands:
     - pytest -v -s kernels/moe/test_moe_layer.py
-
+  mirror:
+    amd:
+      label: ":amd: (MI355) FusedMoE Layer Kernels"
+      dind: false
+      device: mi355_2
+      timeout_in_minutes: 35
+      working_dir: "/vllm-workspace/tests"
+      depends_on:
+      - image-build-amd
+      source_file_dependencies:
+      - csrc/moe/
+      - csrc/rocm/
+      - tests/kernels/moe
+      - vllm/model_executor/layers/fused_moe/
+      - vllm/model_executor/layers/quantization/
+      - vllm/distributed/
+      - vllm/config/
+      - vllm/forward_context.py
+      - vllm/v1/worker/workspace.py
+      - vllm/utils/import_utils.py
+      - vllm/utils/math_utils.py
+      - vllm/utils/torch_utils.py
+      - vllm/platforms/
+      - vllm/_aiter_ops.py
 
 - label: ":nvidia: (B200) FusedMoE Layer Kernels"
   key: kernels-fusedmoe-layer-test-2-b200s
@@ -538,3 +632,90 @@ steps:
   commands:
     - pytest -v -s kernels/moe/test_moe_layer.py
     - pytest -v -s kernels/moe/test_deepep_v2_moe.py
+
+- label: ":amd: (MI355) MLA Kernels"
+  key: amd-mla-kernels-mi355
+  dind: false
+  device: mi355_1
+  num_devices: 1
+  timeout_in_minutes: 25
+  working_dir: /vllm-workspace/
+  depends_on:
+  - image-build-amd
+  source_file_dependencies:
+  - vllm/v1/attention/backends/mla/rocm_aiter_mla.py
+  - vllm/v1/attention/selector.py
+  - vllm/platforms/rocm.py
+  - vllm/_aiter_ops.py
+  - tests/kernels/attention/test_rocm_aiter_mla_decode.py
+  - tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py
+  - tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py
+  - tests/kernels/attention/test_rocm_aiter_mla_op_registration.py
+  commands:
+  - amd-smi
+  - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py
+  - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode.py
+  - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py
+  - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_op_registration.py
+
+- label: ":amd: (MI355) FP8 MoE Kernels"
+  key: amd-fp8-moe-kernels-mi355
+  dind: false
+  device: mi355_1
+  num_devices: 1
+  timeout_in_minutes: 40
+  working_dir: /vllm-workspace/tests
+  depends_on:
+  - image-build-amd
+  source_file_dependencies:
+  - csrc/moe/
+  - vllm/model_executor/layers/fused_moe/
+  - tests/kernels/moe/test_deepep_moe.py
+  - vllm/_aiter_ops.py
+  - vllm/platforms/rocm.py
+  - vllm/envs.py
+  - tests/kernels/moe/test_gpt_oss_triton_kernels.py
+  - tests/kernels/moe/test_modular_oai_triton_moe.py
+  - tests/kernels/moe/test_moe.py
+  - tests/kernels/moe/test_block_int8.py
+  - tests/kernels/moe/test_triton_moe_no_act_mul.py
+  - tests/kernels/moe/test_triton_moe_ptpc_fp8.py
+  commands:
+  - pytest -v -s kernels/moe/test_gpt_oss_triton_kernels.py
+  - pytest -v -s kernels/moe/test_modular_oai_triton_moe.py
+  - pytest -v -s kernels/moe/test_moe.py
+  - pytest -v -s kernels/moe/test_block_int8.py
+  - pytest -v -s kernels/moe/test_triton_moe_no_act_mul.py
+  - pytest -v -s kernels/moe/test_triton_moe_ptpc_fp8.py
+
+- label: ":amd: (MI355) Kernels"
+  key: amd-kernels-mi355
+  dind: false
+  device: mi355_1
+  num_devices: 1
+  timeout_in_minutes: 25
+  working_dir: /vllm-workspace/
+  depends_on:
+  - image-build-amd
+  source_file_dependencies:
+  - csrc/quantization/fp4/
+  - csrc/attention/mla/
+  - csrc/quantization/cutlass_w8a8/moe/
+  - vllm/model_executor/layers/fused_moe/cutlass_moe.py
+  - vllm/v1/attention/backends/triton_attn.py
+  - vllm/v1/attention/backends/rocm_attn.py
+  - vllm/v1/attention/backends/rocm_aiter_fa.py
+  - vllm/v1/attention/backends/rocm_aiter_unified_attn.py
+  - vllm/v1/attention/backends/mla/aiter_triton_mla.py
+  - vllm/v1/attention/backends/mla/rocm_aiter_mla.py
+  - vllm/v1/attention/selector.py
+  - vllm/platforms/rocm.py
+  - vllm/_aiter_ops.py
+  - examples/basic/offline_inference/chat.py
+  - tests/kernels/attention/test_attention_selector.py
+  - tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py
+  commands:
+  - amd-smi
+  - python3 examples/basic/offline_inference/chat.py --attention-backend TRITON_ATTN
+  - pytest -v -s tests/kernels/attention/test_attention_selector.py
+  - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py

.buildkite/test_areas/lm_eval.yaml

@@ -30,6 +30,27 @@ steps:
       - vllm/_aiter_ops.py
       - vllm/platforms/rocm.py
 
+- label: ":amd: (MI300) LM Eval Small Models Harness"
+  key: amd-lm-eval-small-models-harness
+  dind: false
+  device: mi300_1
+  timeout_in_minutes: 35
+  depends_on:
+  - image-build-amd
+  working_dir: /vllm-workspace/.buildkite/lm-eval-harness
+  source_file_dependencies:
+  - csrc/
+  - vllm/model_executor/layers/quantization
+  - vllm/model_executor/models/
+  - vllm/model_executor/model_loader/
+  - vllm/v1/attention/backends/
+  - vllm/v1/attention/selector.py
+  - vllm/_aiter_ops.py
+  - vllm/platforms/rocm.py
+  - .buildkite/lm-eval-harness/
+  commands:
+  - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-small-rocm.txt
+
 - label: ":nvidia: (H200 MIG 35GB) LM Eval Watermarking"
   device: h200_35gb
   key: lm-eval-watermarking
@@ -100,7 +121,7 @@ steps:
       label: ":amd: (MI355) LM Eval Small Models Distributed"
       dind: false
       device: mi355_2
-      timeout_in_minutes: 45
+      timeout_in_minutes: 55
       depends_on:
       - image-build-amd
       source_file_dependencies:
@@ -286,13 +307,16 @@ steps:
       label: ":amd: (MI300) LM Eval Large Models"
       dind: false
       device: mi300_8
-      timeout_in_minutes: 40
+      timeout_in_minutes: 65
       depends_on:
       - image-build-amd
       commands:
       - export VLLM_WORKER_MULTIPROC_METHOD=spawn
       - export PYTORCH_ROCM_ARCH=gfx942 # Limit Quark compilation to save time
       - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx.txt
+      source_file_dependencies:
+      - vllm/_aiter_ops.py
+      - vllm/platforms/rocm.py
 
 - label: ":nvidia: (H100) MoE Refactor Integration TEMPORARY"
   key: moe-refactor-integration-test-h100-temporary

.buildkite/test_areas/lora.yaml

@@ -61,7 +61,7 @@ steps:
       dind: false
       device: mi300_4
       working_dir: "/vllm-workspace/tests"
-      timeout_in_minutes: 25
+      timeout_in_minutes: 65
       depends_on:
       - image-build-amd
       source_file_dependencies:
@@ -73,11 +73,11 @@ steps:
       - export VLLM_WORKER_MULTIPROC_METHOD=spawn
       # expandable_segments breaks custom all-reduce IPC on ROCm.
       - case "$$BUILDKITE_PARALLEL_JOB" in 0|1|2|3) ;; *) echo "unexpected BUILDKITE_PARALLEL_JOB=$$BUILDKITE_PARALLEL_JOB" && exit 1;; esac
-      - test "$$BUILDKITE_PARALLEL_JOB" != "3" || pytest -v -s -x lora/test_chatglm3_tp.py
-      - test "$$BUILDKITE_PARALLEL_JOB" != "1" || pytest -v -s -x lora/test_llama_tp.py
-      - test "$$BUILDKITE_PARALLEL_JOB" != "3" || pytest -v -s -x lora/test_qwen3_with_multi_loras.py
-      - test "$$BUILDKITE_PARALLEL_JOB" != "2" || pytest -v -s -x lora/test_olmoe_tp.py
-      - test "$$BUILDKITE_PARALLEL_JOB" != "0" || pytest -v -s -x lora/test_gptoss_tp.py
-      - test "$$BUILDKITE_PARALLEL_JOB" != "1" || pytest -v -s -x lora/test_qwen35_densemodel_lora.py
-      - test "$$BUILDKITE_PARALLEL_JOB" != "2" || pytest -v -s -x lora/test_gemma4_tp.py
-      - test "$$BUILDKITE_PARALLEL_JOB" != "0" || pytest -v -s -x lora/test_sequence_classification.py::test_batched_loras_tp
+      - if [ "$$BUILDKITE_PARALLEL_JOB" = "3" ]; then pytest -v -s -x lora/test_chatglm3_tp.py; fi
+      - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s -x lora/test_llama_tp.py; fi
+      - if [ "$$BUILDKITE_PARALLEL_JOB" = "3" ]; then pytest -v -s -x lora/test_qwen3_with_multi_loras.py; fi
+      - if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s -x lora/test_olmoe_tp.py; fi
+      - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s -x lora/test_gptoss_tp.py; fi
+      - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s -x lora/test_qwen35_densemodel_lora.py; fi
+      - if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s -x lora/test_gemma4_tp.py; fi
+      - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s -x lora/test_sequence_classification.py::test_batched_loras_tp; fi

.buildkite/test_areas/misc.yaml

@@ -99,7 +99,7 @@ steps:
       label: ":amd: (MI300) V1 Core"
       dind: false
       device: mi300_1
-      timeout_in_minutes: 45
+      timeout_in_minutes: 60
       depends_on:
       - image-build-amd
 
@@ -119,7 +119,7 @@ steps:
       label: ":amd: (MI300) V1 Executor + Worker"
       dind: false
       device: mi300_1
-      timeout_in_minutes: 45
+      timeout_in_minutes: 60
       depends_on:
       - image-build-amd
 
@@ -138,7 +138,7 @@ steps:
       label: ":amd: (MI300) V1 KV Offload"
       dind: false
       device: mi300_1
-      timeout_in_minutes: 45
+      timeout_in_minutes: 60
       depends_on:
       - image-build-amd
 
@@ -158,7 +158,7 @@ steps:
       label: ":amd: (MI300) V1 KV Connectors Shard %N"
       dind: false
       device: mi300_1
-      timeout_in_minutes: 45
+      timeout_in_minutes: 60
       depends_on:
       - image-build-amd
 
@@ -258,7 +258,7 @@ steps:
       label: ":amd: (MI355) Extract Hidden States Integration"
       dind: false
       device: mi355_2
-      timeout_in_minutes: 20
+      timeout_in_minutes: 30
       depends_on:
       - image-build-amd
       source_file_dependencies:
@@ -305,6 +305,15 @@ steps:
   - pip install 'modelscope<1.38'
   - pytest -v -s test_regression.py
   working_dir: "/vllm-workspace/tests" # optional
+  mirror:
+    amd:
+      label: ":amd: (MI355) Regression"
+      dind: false
+      device: mi355_1
+      timeout_in_minutes: 25
+      depends_on:
+      - image-build-amd
+      working_dir: /vllm-workspace/tests
 
 - label: ":nvidia: (H200 MIG 35GB) Examples"
   device: h200_35gb
@@ -446,9 +455,15 @@ steps:
       label: ":amd: (MI355) Async Engine, Inputs, Utils, Worker"
       dind: false
       device: mi355_1
-      timeout_in_minutes: 40
+      timeout_in_minutes: 50
       depends_on:
       - image-build-amd
+      commands:
+      - pytest -v -s detokenizer
+      - pytest -v -s -m 'not cpu_test' multimodal
+      # test_cache.py marks every test cpu_test, including this GPU regression.
+      - pytest -v -s multimodal/test_cache.py::test_sleep_wake_preserves_mm_cache_consistency
+      - pytest -v -s utils_
 
 - label: ":computer: (CPU) Params, Env, Tokenizers, Parser"
   key: cpu-params-env-tokenizers-parser
@@ -572,7 +587,7 @@ steps:
     amd:
       label: ":amd: (MI250) Batch Invariance"
       device: mi250_1
-      timeout_in_minutes: 30
+      timeout_in_minutes: 35
       depends_on:
       - image-build-amd
       source_file_dependencies:
@@ -623,3 +638,17 @@ steps:
   commands:
     - export VLLM_ALLOW_INSECURE_SERIALIZATION=1
     - pytest -v -s v1/spec_decode/test_acceptance_length.py -m slow_test
+  mirror:
+    amd:
+      label: ":amd: (MI355) Acceptance Length (Large Models)"
+      dind: false
+      device: mi355_1
+      timeout_in_minutes: 45
+      depends_on:
+      - image-build-amd
+      working_dir: /vllm-workspace/tests
+      source_file_dependencies:
+      - vllm/v1/spec_decode/
+      - vllm/model_executor/models/mlp_speculator.py
+      - tests/v1/spec_decode/test_acceptance_length.py
+      - vllm/platforms/rocm.py

.buildkite/test_areas/models_basic.yaml

@@ -19,7 +19,7 @@ steps:
       label: ":amd: (MI355) Basic Models (Initialization)"
       dind: false
       device: mi355_1
-      timeout_in_minutes: 25
+      timeout_in_minutes: 30
       depends_on:
       - image-build-amd
 
@@ -37,6 +37,23 @@ steps:
     # test.) Also run if model initialization test file is modified
     - pytest -v -s models/test_initialization.py -k 'not test_can_initialize_small_subset' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
   parallelism: 14
+  mirror:
+    amd:
+      label: ":amd: (MI355) Basic Models (Extra Initialization) Shard %N"
+      dind: false
+      device: mi355_1
+      timeout_in_minutes: 60
+      parallelism: 10
+      working_dir: "/vllm-workspace/tests"
+      depends_on:
+      - image-build-amd
+      source_file_dependencies:
+      - vllm/model_executor/models/
+      - vllm/model_executor/layers/
+      - tests/models/test_initialization.py
+      - tests/models/registry.py
+      - vllm/_aiter_ops.py
+      - vllm/platforms/rocm.py
 
 - label: ":nvidia: (H200 MIG 35GB) Basic Models (Other)"
   device: h200_35gb
@@ -162,6 +179,6 @@ steps:
       device: mi250_1
       no_gpu: true
       working_dir: "/vllm-workspace/tests"
-      timeout_in_minutes: 25
+      timeout_in_minutes: 35
       depends_on:
       - image-build-amd

.buildkite/test_areas/models_distributed.yaml

@@ -38,7 +38,7 @@ steps:
       label: ":amd: (MI300) Distributed Models Shard %N"
       dind: false
       device: mi300_2
-      timeout_in_minutes: 35
+      timeout_in_minutes: 80
       depends_on:
       - image-build-amd
       source_file_dependencies:
@@ -63,7 +63,7 @@ steps:
       - |
         case "$$BUILDKITE_PARALLEL_JOB" in
           0)
-            TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
+            TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' &&
             HIP_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
             ;;
           1)
@@ -72,8 +72,8 @@ steps:
             VLLM_ROCM_USE_AITER=1 pytest -v -s models/test_hyv4_rocm.py -m 'distributed(num_gpus=2)'
             ;;
           2)
-            pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py
-            pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)'
+            pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py &&
+            pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)' &&
             VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)'
             ;;
           *) exit 2 ;;

.buildkite/test_areas/models_language.yaml

@@ -19,7 +19,7 @@ steps:
       label: ":amd: (MI300) Language Models (Standard)"
       dind: false
       device: mi300_1
-      timeout_in_minutes: 45
+      timeout_in_minutes: 60
       depends_on:
       - image-build-amd
 
@@ -82,10 +82,10 @@ steps:
   parallelism: 2
   mirror:
     amd:
-      label: ":amd: (MI300) Language Models (Hybrid) Shard %N"
+      label: ":amd: (MI355) Language Models (Hybrid) Shard %N"
       dind: false
-      device: mi300_1
-      timeout_in_minutes: 60
+      device: mi355_1
+      timeout_in_minutes: 65
       depends_on:
       - image-build-amd
       commands:
@@ -133,7 +133,7 @@ steps:
       label: ":amd: (MI355) Language Models (Extended Generation)"
       dind: false
       device: mi355_1
-      timeout_in_minutes: 70
+      timeout_in_minutes: 95
       depends_on:
       - image-build-amd
       commands:
@@ -197,11 +197,8 @@ steps:
       dind: false
       device: mi300_1
       timeout_in_minutes: 95
-      parallelism: 2
       depends_on:
       - image-build-amd
-      commands:
-      - pytest -v -s models/language/pooling -m 'not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
 
 - label: ":nvidia: (H200 MIG 18GB) Language Models (MTEB)"
   key: language-models-test-mteb

.buildkite/test_areas/models_multimodal.yaml

@@ -77,9 +77,9 @@ steps:
     - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model  # Otherwise, mp_method="spawn" doesn't work
   mirror:
     amd:
-      label: ":amd: (MI300) Multimodal Models (Standard) 4: other + whisper"
+      label: ":amd: (MI355) Multimodal Models (Standard) 4: other + whisper"
       dind: false
-      device: mi300_1
+      device: mi355_1
       timeout_in_minutes: 70
       depends_on:
       - image-build-amd
@@ -104,7 +104,8 @@ steps:
       device: mi250_1
       no_gpu: true
       working_dir: "/vllm-workspace/tests"
-      timeout_in_minutes: 80
+      timeout_in_minutes: 90
+      parallelism: 6
       depends_on:
       - image-build-amd
 

.buildkite/test_areas/plugins.yaml

@@ -75,6 +75,20 @@ steps:
   commands:
     - pip install "vllm-gguf-plugin >= 0.0.2"
     - pytest -v -s plugins_tests/gguf
+  mirror:
+    amd:
+      label: ":amd: (MI355) GGUF Plugin"
+      dind: false
+      device: mi355_1
+      timeout_in_minutes: 45
+      working_dir: "/vllm-workspace/tests"
+      depends_on:
+      - image-build-amd
+      source_file_dependencies:
+      - vllm/model_executor/layers/quantization
+      - tests/plugins_tests/test_gguf_plugin.py
+      - tests/plugins_tests/gguf
+      - vllm/platforms/rocm.py
 
 - label: ":nvidia: (H200 MIG 18GB) BitsAndBytes Plugin"
   key: bitsandbytes-plugin

.buildkite/test_areas/pytorch.yaml

@@ -122,6 +122,21 @@ steps:
     - tests/compile/passes
   commands:
   - pytest -s -v compile/passes --ignore compile/passes/distributed
+  mirror:
+    amd:
+      label: ':amd: (MI300) PyTorch Compilation Passes'
+      dind: false
+      device: mi300_1
+      timeout_in_minutes: 80
+      working_dir: /vllm-workspace/tests
+      depends_on:
+      - image-build-amd
+      source_file_dependencies:
+      - vllm/
+      - tests/compile/passes
+      - vllm/platforms/rocm.py
+      commands:
+      - pytest -s -v compile/passes --ignore compile/passes/distributed
 
 - label: ":nvidia: (H200 MIG 35GB) PyTorch Fullgraph"
   device: h200_35gb
@@ -154,6 +169,7 @@ steps:
   - vllm/utils/
   - vllm/v1/
   - tests/compile
+  - csrc/
   commands:
   # Run smoke tests under fullgraph directory, except test_full_cudagraph.py
   # as it is a heavy test that is covered in other steps.

.buildkite/test_areas/quantization.yaml

@@ -20,6 +20,31 @@ steps:
   # The SM90-only checkpoint currently contains a removed weight_chan_scale
   # parameter. It was not exercised by the previous L4 job.
   - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py -k 'not test_compressed_tensors_w4a8_fp8' --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
+  mirror:
+    amd:
+      label: ":amd: (MI355) Quantization Shard %N"
+      dind: false
+      device: mi355_1
+      timeout_in_minutes: 115
+      working_dir: "/vllm-workspace/tests"
+      depends_on:
+      - image-build-amd
+      source_file_dependencies:
+      - csrc/
+      - vllm/model_executor/layers/quantization
+      - tests/quantization
+      - vllm/_aiter_ops.py
+      - vllm/platforms/rocm.py
+      - vllm/model_executor/layers/fused_moe/oracle/unquantized.py
+      - vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py
+      - tests/rocm/test_moe_weight_replay.py
+      commands:
+      # Use the ROCm default instead of the NVIDIA runner override.
+      - unset VLLM_USE_V2_MODEL_RUNNER
+      - uv pip install --system torchao==0.17.0
+      - uv pip install --system conch-triton-kernels
+      - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
+      - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s rocm/test_moe_weight_replay.py; fi
 
 - label: ":nvidia: (H200 MIG 35GB) Quantized Fusions"
   device: h200_35gb
@@ -63,3 +88,22 @@ steps:
   - tests/models/quantization
   commands:
     - pytest -v -s models/quantization
+  mirror:
+    amd:
+      label: ":amd: (MI355) Quantized Models"
+      dind: false
+      device: mi355_1
+      timeout_in_minutes: 80
+      working_dir: "/vllm-workspace/tests"
+      depends_on:
+      - image-build-amd
+      source_file_dependencies:
+      - vllm/model_executor/layers/quantization
+      - tests/models/quantization
+      - vllm/_aiter_ops.py
+      - vllm/platforms/rocm.py
+      - vllm/model_executor/model_loader/
+      commands:
+      # Use the ROCm default instead of the NVIDIA runner override.
+      - unset VLLM_USE_V2_MODEL_RUNNER
+      - pytest -v -s models/quantization

.buildkite/test_areas/ray_compat.yaml

@@ -16,3 +16,19 @@ steps:
   - setup.py
   commands:
   - bash /vllm-workspace/.buildkite/scripts/check-ray-compatibility.sh
+  mirror:
+    amd:
+      label: ":amd: (MI355) Ray Dependency Compatibility Check"
+      dind: false
+      device: mi355_1
+      timeout_in_minutes: 25
+      working_dir: "/"
+      depends_on:
+      - image-build-amd
+      source_file_dependencies:
+      - .buildkite/scripts/check-ray-compatibility.sh
+      - requirements/
+      - setup.py
+      - vllm/platforms/rocm.py
+      commands:
+      - bash /vllm-workspace/.buildkite/scripts/check-ray-compatibility.sh

.buildkite/test_areas/rust_frontend.yaml

@@ -136,7 +136,7 @@ steps:
       label: ":amd: (MI355) Rust Frontend Core Correctness"
       dind: false
       device: mi355_1
-      timeout_in_minutes: 20
+      timeout_in_minutes: 25
       depends_on:
       - image-build-amd
       source_file_dependencies:
@@ -166,7 +166,7 @@ steps:
       label: ":amd: (MI355) Rust Frontend Tool Use"
       dind: false
       device: mi355_1
-      timeout_in_minutes: 25
+      timeout_in_minutes: 30
       depends_on:
       - image-build-amd
       source_file_dependencies:

.buildkite/test_areas/spec_decode.yaml

@@ -47,7 +47,7 @@ steps:
       label: ":amd: (MI300) Spec Decode Eagle 1: DeepSeek + Qwen"
       dind: false
       device: mi300_1
-      timeout_in_minutes: 35
+      timeout_in_minutes: 60
       depends_on:
       - image-build-amd
       source_file_dependencies:
@@ -74,7 +74,7 @@ steps:
       label: ":amd: (MI300) Spec Decode Eagle 2: Llama 3 + Qwen VL + Other"
       dind: false
       device: mi300_1
-      timeout_in_minutes: 25
+      timeout_in_minutes: 60
       depends_on:
       - image-build-amd
       source_file_dependencies:
@@ -187,7 +187,7 @@ steps:
       label: ":amd: (MI300) Spec Decode Draft Model"
       dind: false
       device: mi300_1
-      timeout_in_minutes: 55
+      timeout_in_minutes: 60
       depends_on:
       - image-build-amd
       source_file_dependencies:
@@ -233,6 +233,9 @@ steps:
       timeout_in_minutes: 40
       depends_on:
       - image-build-amd
+      source_file_dependencies:
+      - vllm/_aiter_ops.py
+      - vllm/platforms/rocm.py
 
 - label: ":nvidia: (B200) Spec Decode DeepSeek MTP Parallel Load"
   key: spec-decode-deepseek-mtp-parallel-load-2xb200-2xmi300
@@ -362,6 +365,33 @@ steps:
     # Each Gemma4 parametrization is an indivisible ~23m floor. Keep them in
     # separate shards and put the remaining tests in shard 0, where new tests
     # land by default.
-    - test "$$BUILDKITE_PARALLEL_JOB" != "0" || pytest -v -s v1/e2e/spec_decode/acceptance_rates/mtp_other/ --deselect "tests/v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]" --deselect "tests/v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]"
-    - test "$$BUILDKITE_PARALLEL_JOB" != "1" || pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]"
-    - test "$$BUILDKITE_PARALLEL_JOB" != "2" || pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]"
+    - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s v1/e2e/spec_decode/acceptance_rates/mtp_other/ --deselect "tests/v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]" --deselect "tests/v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]"; fi
+    - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]"; fi
+    - if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]"; fi
+  mirror:
+    amd:
+      label: ":amd: (MI355) Spec Decode AL MTP + Other Acceptance Nightly"
+      dind: false
+      device: mi355_1
+      timeout_in_minutes: 55
+      depends_on:
+      - image-build-amd
+      working_dir: /vllm-workspace/tests
+      source_file_dependencies:
+      - vllm/v1/spec_decode/
+      - vllm/v1/sample/
+      - vllm/v1/worker/gpu/
+      - vllm/v1/worker/gpu_model_runner.py
+      - vllm/v1/worker/gpu_worker.py
+      - vllm/model_executor/model_loader/
+      - vllm/model_executor/models/gemma4.py
+      - vllm/model_executor/models/gemma4_mm.py
+      - vllm/model_executor/models/gemma4_mtp.py
+      - vllm/model_executor/models/llama.py
+      - vllm/model_executor/models/llama_eagle3.py
+      - vllm/model_executor/models/medusa.py
+      - vllm/model_executor/models/registry.py
+      - tests/evals/gsm8k/
+      - tests/utils.py
+      - tests/v1/e2e/spec_decode/
+      - vllm/platforms/rocm.py

.buildkite/test_areas/weight_loading.yaml

@@ -20,7 +20,7 @@ steps:
       label: ":amd: (MI355) Weight Loading Multi-GPU"
       dind: false
       device: mi355_2
-      timeout_in_minutes: 40
+      timeout_in_minutes: 70
       depends_on:
       - image-build-amd
       commands: