[CI][ROCm] Add ten AMD parity groups (#57876)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>

.buildkite/test-amd.yaml

@@ -1361,6 +1361,23 @@ steps:
   - bash ../tools/install_torchcodec_rocm.sh || exit 1
   - pytest -s entrypoints/openai/correctness/
 
+- label: ":amd: (MI300) Scale-out EC Connector E2E"
+  timeout_in_minutes: 60
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
+  agent_pool: mi300_2
+  num_gpus: 2
+  optional: true
+  working_dir: "/vllm-workspace/tests"
+  source_file_dependencies:
+  - vllm/distributed/ec_transfer/
+  - vllm/entrypoints/scale_out/
+  - tests/entrypoints/scale_out/ec_integration/
+  - vllm/platforms/rocm.py
+  commands:
+  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+  - bash entrypoints/scale_out/ec_integration/run_scale_out_ec_e2e_test.sh
+
 #-----------------------------------------------------------  mi300 · evals  -----------------------------------------------------------#
 
 - label: ":amd: (MI300) DeepSeek V2-Lite Prefetch Offload Accuracy" # TBD
@@ -3190,6 +3207,58 @@ steps:
       v1/e2e/spec_decode/draft_model/test_draft_model.py::test_draft_model_tensor_parallelism
       v1/e2e/spec_decode/draft_model/test_draft_model.py::test_draft_model_engine_args_tensor_parallelism
 
+- label: ":amd: (MI300) Mooncake EC TCP E2E"
+  timeout_in_minutes: 60
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
+  agent_pool: mi300_2
+  num_gpus: 2
+  optional: true
+  working_dir: "/vllm-workspace"
+  source_file_dependencies:
+  - vllm/distributed/ec_transfer/
+  - vllm/config/ec_transfer.py
+  - vllm/config/multimodal.py
+  - vllm/config/vllm.py
+  - vllm/multimodal/
+  - vllm/v1/core/encoder_cache_manager.py
+  - vllm/v1/core/sched/
+  - vllm/v1/engine/
+  - vllm/v1/worker/ec_connector_model_runner_mixin.py
+  - vllm/v1/worker/gpu_model_runner.py
+  - vllm/v1/worker/gpu_worker.py
+  - vllm/v1/worker/gpu/ec_connector.py
+  - vllm/v1/worker/gpu/model_runner.py
+  - vllm/v1/worker/mm_encoder_model_runner.py
+  - vllm/v1/worker/encoder_cudagraph.py
+  - vllm/v1/worker/encoder_cudagraph_defs.py
+  - vllm/v1/attention/ops/vit_attn_wrappers.py
+  - vllm/v1/worker/gpu/model_states/
+  - vllm/v1/worker/gpu/mm/
+  - tests/v1/cudagraph/test_encoder_cudagraph.py
+  - tests/models/multimodal/generation/test_vit_cudagraph.py
+  - examples/disaggregated/disaggregated_encoder/
+  - tests/v1/ec_connector/
+  - requirements/kv_connectors.txt
+  - vllm/platforms/rocm.py
+  - requirements/rocm.txt
+  commands:
+  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+  - export PYTHON_BIN='python'
+  - export PYTHONUNBUFFERED='1'
+  - export VLLM_HOST_IP='127.0.0.1'
+  - export VLLM_USE_V2_MODEL_RUNNER='1'
+  - export E_CUDAGRAPH_MM_ENCODER='1'
+  - export E_MM_ENCODER_ATTN_BACKEND='TRITON_ATTN'
+  - export MOONCAKE_EC_PROTOCOL='tcp'
+  - export USE_MM_PROMPTS='1'
+  - export SKIP_BASELINE='0'
+  - export CONCURRENCY='3'
+  - export REPEAT='2'
+  - export LOG_PATH='/tmp/mooncake-ec-e2e'
+  - export BASELINE_FILE='/tmp/mooncake-ec-e2e/baseline.json'
+  - bash tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh
+
 - label: ":amd: (MI300) CrossLayer KV layout Distributed NixlConnector PD accuracy" # TBD
   timeout_in_minutes: 60
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
@@ -3385,6 +3454,27 @@ steps:
 
 #--------------------------------------------------------  mi355 · distributed  --------------------------------------------------------#
 
+- label: ":amd: (MI355) Sharded RDT Weight Transfer"
+  timeout_in_minutes: 45
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
+  dind: false
+  agent_pool: mi355_1
+  num_gpus: 1
+  optional: true
+  working_dir: "/vllm-workspace/tests"
+  source_file_dependencies:
+  - vllm/distributed/weight_transfer/
+  - tests/distributed/test_sharded_rdt_plan.py
+  - tests/distributed/test_sharded_rdt_producer.py
+  - tests/distributed/test_sharded_rdt_trainer.py
+  - vllm/platforms/rocm.py
+  commands:
+  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+  - uv pip install --system --no-deps 'ray==2.56.1'
+  - pytest -v -s distributed/test_sharded_rdt_plan.py
+  - pytest -v -s distributed/test_sharded_rdt_producer.py
+  - pytest -v -s distributed/test_sharded_rdt_trainer.py
+
 - label: ":amd: (MI355) Distributed DP Basic" # TBD
   timeout_in_minutes: 45
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
@@ -3464,6 +3554,35 @@ steps:
   - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
   - DP_EP=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
 
+- label: ":amd: (MI355) Kimi-Linear-48B-A3B Disaggregated DP EP"
+  timeout_in_minutes: 105
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
+  dind: false
+  agent_pool: mi355_4
+  num_gpus: 4
+  optional: true
+  working_dir: "/vllm-workspace/tests"
+  source_file_dependencies:
+  - vllm/platforms/rocm.py
+  - vllm/models/kimi_k3/
+  - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
+  - tests/v1/kv_connector/nixl_integration/
+  - requirements/kv_connectors_rocm.txt
+  commands:
+  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+  - export ENABLE_HMA_FLAG='1'
+  - export DP_EP='1'
+  - export GPU_MEMORY_UTILIZATION='0.9'
+  - export VLLM_SSM_CONV_STATE_LAYOUT='DS'
+  - export PREFILLER_TP_SIZE='2'
+  - export DECODER_TP_SIZE='2'
+  - export MODEL_NAMES='moonshotai/Kimi-Linear-48B-A3B-Instruct'
+  - export PREFILL_BLOCK_SIZE='2048'
+  - export DECODE_BLOCK_SIZE='2048'
+  - export VLLM_SERVE_EXTRA_ARGS='--trust-remote-code,--mamba-cache-mode align,--max-model-len 32768'
+  - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
+  - bash v1/kv_connector/nixl_integration/run_accuracy_test.sh
+
 - label: ":amd: (MI355) Distributed Features" # TBD
   timeout_in_minutes: 90
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
@@ -3661,6 +3780,51 @@ steps:
     - uv pip install --system 'gpt-oss[eval]==0.0.5'
     - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt
 
+- label: ":amd: (MI355) LM Eval Watermarking"
+  timeout_in_minutes: 45
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
+  dind: false
+  agent_pool: mi355_1
+  num_gpus: 1
+  optional: true
+  working_dir: "/vllm-workspace/tests"
+  source_file_dependencies:
+  - .buildkite/test_areas/lm_eval.yaml
+  - tests/evals/gsm8k/
+  - vllm/config/watermarking.py
+  - vllm/sampling_params.py
+  - vllm/v1/watermarking/
+  - vllm/v1/worker/gpu/
+  - vllm/platforms/rocm.py
+  commands:
+  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-watermark.txt
+
+- label: ":amd: (MI355) LM Eval Watermark Feature Combination"
+  timeout_in_minutes: 60
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
+  dind: false
+  agent_pool: mi355_2
+  num_gpus: 2
+  optional: true
+  working_dir: "/vllm-workspace/tests"
+  source_file_dependencies:
+  - .buildkite/test_areas/lm_eval.yaml
+  - tests/evals/gsm8k/
+  - vllm/config/speculative.py
+  - vllm/config/vllm.py
+  - vllm/config/watermarking.py
+  - vllm/sampling_params.py
+  - vllm/model_executor/models/qwen3_dspark.py
+  - vllm/model_executor/models/qwen3_dflash.py
+  - vllm/model_executor/warmup/
+  - vllm/v1/watermarking/
+  - vllm/v1/worker/gpu/
+  - vllm/platforms/rocm.py
+  commands:
+  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-watermark-dspark-tp2.txt
+
 - label: ":amd: (MI355) LM Eval Qwen3-5 Models" # TBD
   timeout_in_minutes: 40
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
@@ -3684,6 +3848,27 @@ steps:
   commands:
   - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-mi355.txt
 
+- label: ":amd: (MI355) Qwen3-8-Flash-Next-FP8 Accuracy Eval"
+  timeout_in_minutes: 105
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
+  dind: false
+  agent_pool: mi355_4
+  num_gpus: 4
+  optional: true
+  working_dir: "/vllm-workspace/tests"
+  source_file_dependencies:
+  - csrc/libtorch_stable/gdn/
+  - tests/evals/qwen4_exp/
+  - vllm/model_executor/layers/mamba/
+  - vllm/models/qwen4_exp/
+  - vllm/transformers_utils/configs/qwen4_exp.py
+  - vllm/v1/attention/backends/short_conv_attn.py
+  - vllm/platforms/rocm.py
+  commands:
+  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+  - uv pip install --system 'evalscope==1.10.0'
+  - pytest -s -v evals/qwen4_exp/test_accuracy.py --config-list-file=configs/models-rocm.txt
+
 - label: ":amd: (MI355) LM Eval Small Models Distributed"
   timeout_in_minutes: 55
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
@@ -3705,6 +3890,25 @@ steps:
   commands:
   - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt
 
+- label: ":amd: (MI355) MoE Refactor Integration Shard %N"
+  timeout_in_minutes: 105
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
+  dind: false
+  agent_pool: mi355_2
+  num_gpus: 2
+  parallelism: 4
+  optional: true
+  working_dir: "/vllm-workspace/tests"
+  source_file_dependencies:
+  - vllm/platforms/rocm.py
+  - tests/evals/gsm8k/
+  - vllm/model_executor/layers/fused_moe/
+  - vllm/model_executor/layers/quantization/
+  - vllm/_aiter_ops.py
+  commands:
+  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor/config-rocm-shard-$$BUILDKITE_PARALLEL_JOB.txt
+
 - label: ":amd: (MI355) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy" # TBD
   timeout_in_minutes: 25
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
@@ -3925,6 +4129,22 @@ steps:
   - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py
   - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_op_registration.py
 
+- label: ":amd: (MI355) DeepSeek V4 Fused Kernels"
+  timeout_in_minutes: 60
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
+  dind: false
+  agent_pool: mi355_1
+  num_gpus: 1
+  optional: true
+  working_dir: "/vllm-workspace/tests"
+  source_file_dependencies:
+  - csrc/libtorch_stable/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu
+  - vllm/models/deepseek_v4/common/ops/
+  - tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py
+  - vllm/platforms/rocm.py
+  commands:
+  - pytest -v -s kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py
+
 - label: ":amd: (MI355) Attention Kernels Shard %N" # TBD
   timeout_in_minutes: 90
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
@@ -4418,6 +4638,25 @@ steps:
   - pytest -v -s v1/e2e/general/test_kv_sharing_fast_prefill.py
   - pytest -v -s v1/e2e/general/test_mamba_prefix_cache.py::test_mamba_prefix_cache_mrv2 v1/e2e/general/test_mamba_prefix_cache.py::test_mamba_prefix_cache_mrv2_async
 
+- label: ":amd: (MI355) Extract Hidden States Integration Single GPU"
+  timeout_in_minutes: 45
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
+  dind: false
+  agent_pool: mi355_1
+  num_gpus: 1
+  optional: true
+  working_dir: "/vllm-workspace"
+  source_file_dependencies:
+  - vllm/v1/spec_decode/extract_hidden_states.py
+  - vllm/model_executor/models/extract_hidden_states.py
+  - vllm/transformers_utils/configs/extract_hidden_states.py
+  - vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py
+  - tests/v1/kv_connector/extract_hidden_states_integration
+  - vllm/platforms/rocm.py
+  commands:
+  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+  - pytest -v -s -m 'not distributed' tests/v1/kv_connector/extract_hidden_states_integration
+
 - label: ":amd: (MI355) V1 Attention Shard %N" # TBD
   timeout_in_minutes: 125
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]

.buildkite/test_areas/disaggregated.yaml

@@ -318,3 +318,24 @@ steps:
   commands:
     - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
     - bash v1/kv_connector/nixl_integration/run_accuracy_test.sh
+  mirror:
+    amd:
+      label: ":amd: (MI355) Kimi-Linear-48B-A3B Disaggregated DP EP"
+      dind: false
+      device: mi355_4
+      num_devices: 4
+      timeout_in_minutes: 105
+      working_dir: /vllm-workspace/tests
+      depends_on:
+      - image-build-amd
+      source_file_dependencies:
+      - vllm/platforms/rocm.py
+      - vllm/models/kimi_k3/
+      - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
+      - tests/v1/kv_connector/nixl_integration/
+      - requirements/kv_connectors_rocm.txt
+      env:
+        VLLM_WORKER_MULTIPROC_METHOD: "spawn"
+      commands:
+      - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
+      - bash v1/kv_connector/nixl_integration/run_accuracy_test.sh

.buildkite/test_areas/disaggregated_ec.yaml

@@ -14,3 +14,22 @@ steps:
     - tests/entrypoints/scale_out/ec_integration/
   commands:
     - bash entrypoints/scale_out/ec_integration/run_scale_out_ec_e2e_test.sh
+  mirror:
+    amd:
+      label: ":amd: (MI300) Scale-out EC Connector E2E"
+      dind: false
+      device: mi300_2
+      num_devices: 2
+      timeout_in_minutes: 60
+      working_dir: /vllm-workspace/tests
+      depends_on:
+      - image-build-amd
+      source_file_dependencies:
+      - vllm/distributed/ec_transfer/
+      - vllm/entrypoints/scale_out/
+      - tests/entrypoints/scale_out/ec_integration/
+      - vllm/platforms/rocm.py
+      env:
+        VLLM_WORKER_MULTIPROC_METHOD: "spawn"
+      commands:
+      - bash entrypoints/scale_out/ec_integration/run_scale_out_ec_e2e_test.sh

.buildkite/test_areas/disaggregated_mooncake.yaml

@@ -83,3 +83,45 @@ steps:
       PY
       )"
     - bash tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh
+  mirror:
+    amd:
+      label: ":amd: (MI300) Mooncake EC TCP E2E"
+      dind: false
+      device: mi300_2
+      num_devices: 2
+      timeout_in_minutes: 60
+      working_dir: /vllm-workspace
+      depends_on:
+      - image-build-amd
+      source_file_dependencies:
+      - vllm/distributed/ec_transfer/
+      - vllm/config/ec_transfer.py
+      - vllm/config/multimodal.py
+      - vllm/config/vllm.py
+      - vllm/multimodal/
+      - vllm/v1/core/encoder_cache_manager.py
+      - vllm/v1/core/sched/
+      - vllm/v1/engine/
+      - vllm/v1/worker/ec_connector_model_runner_mixin.py
+      - vllm/v1/worker/gpu_model_runner.py
+      - vllm/v1/worker/gpu_worker.py
+      - vllm/v1/worker/gpu/ec_connector.py
+      - vllm/v1/worker/gpu/model_runner.py
+      - vllm/v1/worker/mm_encoder_model_runner.py
+      - vllm/v1/worker/encoder_cudagraph.py
+      - vllm/v1/worker/encoder_cudagraph_defs.py
+      - vllm/v1/attention/ops/vit_attn_wrappers.py
+      - vllm/v1/worker/gpu/model_states/
+      - vllm/v1/worker/gpu/mm/
+      - tests/v1/cudagraph/test_encoder_cudagraph.py
+      - tests/models/multimodal/generation/test_vit_cudagraph.py
+      - examples/disaggregated/disaggregated_encoder/
+      - tests/v1/ec_connector/
+      - requirements/kv_connectors.txt
+      - vllm/platforms/rocm.py
+      - requirements/rocm.txt
+      env:
+        VLLM_WORKER_MULTIPROC_METHOD: "spawn"
+        PYTHON_BIN: "python"
+      commands:
+      - bash tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh

.buildkite/test_areas/distributed.yaml

@@ -33,6 +33,29 @@ steps:
   - pytest -v -s distributed/test_sharded_rdt_plan.py
   - pytest -v -s distributed/test_sharded_rdt_producer.py
   - pytest -v -s distributed/test_sharded_rdt_trainer.py
+  mirror:
+    amd:
+      label: ":amd: (MI355) Sharded RDT Weight Transfer"
+      dind: false
+      device: mi355_1
+      num_devices: 1
+      timeout_in_minutes: 45
+      working_dir: /vllm-workspace/tests
+      depends_on:
+      - image-build-amd
+      source_file_dependencies:
+      - vllm/distributed/weight_transfer/
+      - tests/distributed/test_sharded_rdt_plan.py
+      - tests/distributed/test_sharded_rdt_producer.py
+      - tests/distributed/test_sharded_rdt_trainer.py
+      - vllm/platforms/rocm.py
+      env:
+        VLLM_WORKER_MULTIPROC_METHOD: "spawn"
+      commands:
+      - uv pip install --system --no-deps 'ray==2.56.1'
+      - pytest -v -s distributed/test_sharded_rdt_plan.py
+      - pytest -v -s distributed/test_sharded_rdt_producer.py
+      - pytest -v -s distributed/test_sharded_rdt_trainer.py
 
 - label: ":nvidia: (L4) Distributed DP Basic"
   key: distributed-dp-tests-2-gpus

.buildkite/test_areas/kernels.yaml

@@ -111,6 +111,23 @@ steps:
   commands:
     - pytest -v -s kernels/test_fused_deepseek_v4_*.py
     - pytest -v -s kernels/test_top_k_per_row.py
+  mirror:
+    amd:
+      label: ":amd: (MI355) DeepSeek V4 Fused Kernels"
+      dind: false
+      device: mi355_1
+      num_devices: 1
+      timeout_in_minutes: 60
+      working_dir: /vllm-workspace/tests
+      depends_on:
+      - image-build-amd
+      source_file_dependencies:
+      - csrc/libtorch_stable/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu
+      - vllm/models/deepseek_v4/common/ops/
+      - tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py
+      - vllm/platforms/rocm.py
+      commands:
+      - pytest -v -s kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py
 
 - label: ":nvidia: (B200) DeepSeek V4 Kernels"
   key: deepseek-v4-kernel-test-b200

.buildkite/test_areas/lm_eval.yaml

@@ -66,6 +66,28 @@ steps:
   autorun_on_main: true
   commands:
   - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-watermark.txt
+  mirror:
+    amd:
+      label: ":amd: (MI355) LM Eval Watermarking"
+      dind: false
+      device: mi355_1
+      num_devices: 1
+      timeout_in_minutes: 45
+      working_dir: /vllm-workspace/tests
+      depends_on:
+      - image-build-amd
+      source_file_dependencies:
+      - .buildkite/test_areas/lm_eval.yaml
+      - tests/evals/gsm8k/
+      - vllm/config/watermarking.py
+      - vllm/sampling_params.py
+      - vllm/v1/watermarking/
+      - vllm/v1/worker/gpu/
+      - vllm/platforms/rocm.py
+      env:
+        VLLM_WORKER_MULTIPROC_METHOD: "spawn"
+      commands:
+      - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-watermark.txt
 
 # - label: LM Eval Large Models (4xA100)
 #   key: lm-eval-large-models-4xa100
@@ -203,6 +225,33 @@ steps:
   autorun_on_main: true
   commands:
   - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-watermark-dspark-tp2.txt
+  mirror:
+    amd:
+      label: ":amd: (MI355) LM Eval Watermark Feature Combination"
+      dind: false
+      device: mi355_2
+      num_devices: 2
+      timeout_in_minutes: 60
+      working_dir: /vllm-workspace/tests
+      depends_on:
+      - image-build-amd
+      source_file_dependencies:
+      - .buildkite/test_areas/lm_eval.yaml
+      - tests/evals/gsm8k/
+      - vllm/config/speculative.py
+      - vllm/config/vllm.py
+      - vllm/config/watermarking.py
+      - vllm/sampling_params.py
+      - vllm/model_executor/models/qwen3_dspark.py
+      - vllm/model_executor/models/qwen3_dflash.py
+      - vllm/model_executor/warmup/
+      - vllm/v1/watermarking/
+      - vllm/v1/worker/gpu/
+      - vllm/platforms/rocm.py
+      env:
+        VLLM_WORKER_MULTIPROC_METHOD: "spawn"
+      commands:
+      - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-watermark-dspark-tp2.txt
 
 - label: ":nvidia: (B200) LM Eval Spec Decode"
   key: lm-eval-spec-decode-4xb200
@@ -329,7 +378,27 @@ steps:
     # build 83851): shards ~12.4/13.6/12.4/14.5m of eval time. New configs must
     # be added to exactly one config-h100-shard-*.txt.
     - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor/config-h100-shard-$$BUILDKITE_PARALLEL_JOB.txt
-  
+  mirror:
+    amd:
+      label: ":amd: (MI355) MoE Refactor Integration Shard %N"
+      dind: false
+      device: mi355_2
+      num_devices: 2
+      timeout_in_minutes: 105
+      working_dir: /vllm-workspace/tests
+      depends_on:
+      - image-build-amd
+      source_file_dependencies:
+      - vllm/platforms/rocm.py
+      - tests/evals/gsm8k/
+      - vllm/model_executor/layers/fused_moe/
+      - vllm/model_executor/layers/quantization/
+      - vllm/_aiter_ops.py
+      env:
+        VLLM_WORKER_MULTIPROC_METHOD: "spawn"
+      commands:
+      - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor/config-rocm-shard-$$BUILDKITE_PARALLEL_JOB.txt
+
 - label: ":nvidia: (B200) MoE Refactor Integration TEMPORARY Shard %N"
   key: moe-refactor-integration-test-b200-temporary
   device: b200-k8s
@@ -578,6 +647,29 @@ steps:
   commands:
     - uv pip install --system 'evalscope==1.10.0'
     - pytest -s -v evals/qwen4_exp/test_accuracy.py --config-list-file=configs/models-h200.txt
+  mirror:
+    amd:
+      label: ":amd: (MI355) Qwen3.8-Flash-Next-FP8 Accuracy Eval"
+      dind: false
+      device: mi355_4
+      num_devices: 4
+      timeout_in_minutes: 105
+      working_dir: /vllm-workspace/tests
+      depends_on:
+      - image-build-amd
+      source_file_dependencies:
+      - csrc/libtorch_stable/gdn/
+      - tests/evals/qwen4_exp/
+      - vllm/model_executor/layers/mamba/
+      - vllm/models/qwen4_exp/
+      - vllm/transformers_utils/configs/qwen4_exp.py
+      - vllm/v1/attention/backends/short_conv_attn.py
+      - vllm/platforms/rocm.py
+      env:
+        VLLM_WORKER_MULTIPROC_METHOD: "spawn"
+      commands:
+      - uv pip install --system 'evalscope==1.10.0'
+      - pytest -s -v evals/qwen4_exp/test_accuracy.py --config-list-file=configs/models-rocm.txt
 
 - label: ":nvidia: (DGX) Spark GPQA Eval (GPT-OSS)"
   key: gpqa-eval-gpt-oss-spark

.buildkite/test_areas/misc.yaml

@@ -237,6 +237,27 @@ steps:
   commands:
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
     - pytest -v -s tests/v1/kv_connector/extract_hidden_states_integration
+  mirror:
+    amd:
+      label: ":amd: (MI355) Extract Hidden States Integration Single GPU"
+      dind: false
+      device: mi355_1
+      num_devices: 1
+      timeout_in_minutes: 45
+      working_dir: /vllm-workspace
+      depends_on:
+      - image-build-amd
+      source_file_dependencies:
+      - vllm/v1/spec_decode/extract_hidden_states.py
+      - vllm/model_executor/models/extract_hidden_states.py
+      - vllm/transformers_utils/configs/extract_hidden_states.py
+      - vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py
+      - tests/v1/kv_connector/extract_hidden_states_integration
+      - vllm/platforms/rocm.py
+      env:
+        VLLM_WORKER_MULTIPROC_METHOD: "spawn"
+      commands:
+      - pytest -v -s -m 'not distributed' tests/v1/kv_connector/extract_hidden_states_integration
 
 - label: ":nvidia: (L4) Extract Hidden States Integration"
   key: extract-hidden-states-integration-2-gpus

tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-BF16-triton-rocm.yaml

@@ -0,0 +1,5 @@
+model_name: "Qwen/Qwen3-30B-A3B"
+accuracy_threshold: 0.88
+num_questions: 1319
+num_fewshot: 5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --moe-backend=triton"

tests/evals/gsm8k/configs/moe-refactor/config-rocm-shard-0.txt

@@ -0,0 +1 @@
+Qwen3-30B-A3B-BF16-triton-rocm.yaml

tests/evals/gsm8k/configs/moe-refactor/config-rocm-shard-1.txt

@@ -0,0 +1 @@
+Qwen3-30B-A3B-Fp8-AutoFp8-triton.yaml

tests/evals/gsm8k/configs/moe-refactor/config-rocm-shard-2.txt

@@ -0,0 +1 @@
+Qwen3-30B-A3B-Fp8-CT-Block-triton.yaml

tests/evals/gsm8k/configs/moe-refactor/config-rocm-shard-3.txt

@@ -0,0 +1 @@
+Mixtral-8x7B-Fp8-AutoFp8-triton.yaml

tests/evals/qwen4_exp/configs/Qwen3.8-Flash-Next-FP8-rocm.yaml

@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+model_name: "Qwen/Qwen3.8-Flash-Next-FP8"
+datasets:
+  gsm8k:
+    metric_threshold: 0.98
+    tolerance: 0.02
+  aime25:
+    metric_threshold: 0.90
+    tolerance: 0.05
+eval_batch_size: 32
+startup_max_wait_seconds: 1800
+generation_config:
+  seed: 1236
+  do_sample: true
+  temperature: 0.7
+  top_p: 0.8
+  top_k: 20
+  repetition_penalty: 1.0
+  presence_penalty: 1.5
+  max_tokens: 32768
+  stream: true
+  extra_body:
+    chat_template_kwargs:
+      enable_thinking: true
+server_args: "--tensor-parallel-size 4 --mamba-cache-mode align"

tests/evals/qwen4_exp/configs/models-rocm.txt

@@ -0,0 +1 @@
+Qwen3.8-Flash-Next-FP8-rocm.yaml

tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py

@@ -96,7 +96,7 @@ def apply_rope_gptj_last_k(
     # `e*c - o*s` fused form. This keeps the reference close to the kernel, but
     # the fp32 reference and the fp32 GPU kernel can still round to bf16 on
     # opposite sides of a round-to-nearest tie for a tiny number of elements at
-    # high positions, so callers compare the RoPE region within 1 bf16 ULP.
+    # high positions, so callers allow for bf16 rounding in the RoPE region.
     new_even = torch.addcmul(-odd * sin, even, cos)
     new_odd = torch.addcmul(odd * cos, even, sin)
     rope_rotated = torch.stack((new_even, new_odd), dim=-1).reshape(shape)
@@ -314,7 +314,19 @@ def test_q_path_without_qnorm_matches_rope_only_reference(num_tokens: int):
         apply_q_norm=False,
     )
 
-    torch.testing.assert_close(q_out[:, :n_heads], q_ref, rtol=0, atol=0)
+    torch.testing.assert_close(
+        q_out[:, :n_heads, :NOPE_DIM], q_ref[..., :NOPE_DIM], rtol=0, atol=0
+    )
+    # HIP and PyTorch can contract different FP32 products into an FMA.
+    # Allow bf16 rounding and small cancellation errors only in the RoPE region
+    # on ROCm; the unrotated dimensions and padded heads must remain exact.
+    rope_rtol, rope_atol = (1e-2, 1e-6) if current_platform.is_rocm() else (0, 0)
+    torch.testing.assert_close(
+        q_out[:, :n_heads, NOPE_DIM:],
+        q_ref[..., NOPE_DIM:],
+        rtol=rope_rtol,
+        atol=rope_atol,
+    )
     assert q_out[:, n_heads:].count_nonzero().item() == 0