[ROCm][DI][CI] Enable WideEP Intranode tests (#53195)

Signed-off-by: lcskrishna <lollachaitanya@gmail.com>
2026-09-08 17:06:36 -05:00
parent 05e52f6336
commit 28a2ccee78
4 changed files with 144 additions and 55 deletions
@@ -37,14 +37,14 @@ models:
      VLLM_ROCM_USE_AITER_MOE: "1"
      VLLM_ROCM_USE_AITER_RMSNORM: "1"
      VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: "0"
+      VLLM_ROCM_USE_AITER_FP8BMM: "false"
    base_flags: "--trust-remote-code --kv-cache-dtype fp8"
    prefill:
      tp: "--gpu-memory-utilization 0.85"
      ep: "--gpu-memory-utilization 0.85 --enforce-eager"
    decode:
      tp: "--gpu-memory-utilization 0.85"
-      #ep: '--gpu-memory-utilization 0.75 --enforce-eager'
-      ep: '--gpu-memory-utilization 0.75 --compilation-config {"cudagraph_mode":"PIECEWISE","custom_ops":["+quant_fp8"]}'
+      ep: '--gpu-memory-utilization 0.75 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8"]}'

  - model: MiniMax-M3-MXFP8
    env:
@@ -53,11 +53,14 @@ models:
      VLLM_ROCM_USE_AITER_RMSNORM: "1"
      VLLM_USE_BREAKABLE_CUDAGRAPH: "0"
      VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: "INT6"
+      VLLM_KV_CACHE_LAYOUT: "HND"
    base_flags: "--trust-remote-code --attention-backend TRITON_ATTN --block-size 128 --language-model-only --kv-cache-dtype fp8"
    prefill:
      tp: "--gpu-memory-utilization 0.85 --enforce-eager"
+      ep: "--gpu-memory-utilization 0.85 --enforce-eager"
    decode:
      tp: "--gpu-memory-utilization 0.85"
+      ep: '--gpu-memory-utilization 0.75 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8"]}'

  - model: DeepSeek-R1-MXFP4
    env:
@@ -65,11 +68,14 @@ models:
      VLLM_ROCM_USE_AITER_MLA: "1"
      VLLM_ROCM_USE_AITER_MOE: "1"
      VLLM_ROCM_USE_AITER_RMSNORM: "1"
+      VLLM_ROCM_USE_AITER_FP8BMM: "false"
    base_flags: "--trust-remote-code --kv-cache-dtype fp8"
    prefill:
      tp: "--gpu-memory-utilization 0.85"
+      ep: "--gpu-memory-utilization 0.85 --enforce-eager"
    decode:
      tp: "--gpu-memory-utilization 0.85"
+      ep: '--gpu-memory-utilization 0.75 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8"]}'

  - model: Kimi-K2.5-MXFP4
    env:
@@ -79,8 +85,10 @@ models:
    base_flags: "--trust-remote-code"
    prefill:
      tp: "--gpu-memory-utilization 0.85"
+      ep: "--gpu-memory-utilization 0.85 --enforce-eager"
    decode:
      tp: "--gpu-memory-utilization 0.85"
+      ep: '--gpu-memory-utilization 0.75 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8"]}'

  - model: Kimi-K2.6-MXFP4
    env:
@@ -89,9 +97,10 @@ models:
      VLLM_ROCM_USE_AITER_MLA: "1"
      VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: "INT4"
      VLLM_ROCM_USE_SKINNY_GEMM: "0"
-      VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: "1"
    base_flags: "--trust-remote-code --kv-cache-dtype fp8 --mm-encoder-tp-mode data --block-size 1 --attention-backend ROCM_AITER_MLA"
    prefill:
      tp: "--gpu-memory-utilization 0.9"
+      ep: "--gpu-memory-utilization 0.85 --enforce-eager"
    decode:
      tp: "--gpu-memory-utilization 0.9"
+      ep: '--gpu-memory-utilization 0.75 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8"]}'
@@ -11,192 +11,273 @@ steps:
 - label: "DeepSeek-V3-PD-1P1D-TP8-MoRIIO-proxy"
  agents:
    queue: amd_mi350_ainic
-  timeout_in_minutes: 120
+  timeout_in_minutes: 180
  concurrency: 2
  concurrency_group: "amd-disagg/mi350/2node"
  commands:
-  - MODEL_NAME=DeepSeek-V3 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+  - MODEL_NAME=DeepSeek-V3 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh

 - label: "DeepSeek-V3-PD-1P1D-TP8-MoRIIO-vllm-router"
  agents:
    queue: amd_mi350_ainic
-  timeout_in_minutes: 120
+  timeout_in_minutes: 180
  concurrency: 2
  concurrency_group: "amd-disagg/mi350/2node"
  commands:
-  - MODEL_NAME=DeepSeek-V3 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+  - MODEL_NAME=DeepSeek-V3 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh

 - label: "DeepSeek-R1-MXFP4-PD-1P1D-TP8-MoRIIO-proxy"
  agents:
    queue: amd_mi350_ainic
-  timeout_in_minutes: 120
+  timeout_in_minutes: 180
  concurrency: 2
  concurrency_group: "amd-disagg/mi350/2node"
  commands:
-  - MODEL_NAME=DeepSeek-R1-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+  - MODEL_NAME=DeepSeek-R1-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh

 - label: "DeepSeek-R1-MXFP4-PD-1P1D-TP8-MoRIIO-vllm-router"
  agents:
    queue: amd_mi350_ainic
-  timeout_in_minutes: 120
+  timeout_in_minutes: 180
  concurrency: 2
  concurrency_group: "amd-disagg/mi350/2node"
  commands:
-  - MODEL_NAME=DeepSeek-R1-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+  - MODEL_NAME=DeepSeek-R1-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh

 - label: "Kimi-K2.5-MXFP4-PD-1P1D-TP8-MoRIIO-proxy"
  agents:
    queue: amd_mi350_ainic
-  timeout_in_minutes: 120
+  timeout_in_minutes: 180
  concurrency: 2
  concurrency_group: "amd-disagg/mi350/2node"
  commands:
-  - MODEL_NAME=Kimi-K2.5-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+  - MODEL_NAME=Kimi-K2.5-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh

 - label: "Kimi-K2.5-MXFP4-PD-1P1D-TP8-MoRIIO-vllm-router"
  agents:
    queue: amd_mi350_ainic
-  timeout_in_minutes: 120
+  timeout_in_minutes: 180
  concurrency: 2
  concurrency_group: "amd-disagg/mi350/2node"
  commands:
-  - MODEL_NAME=Kimi-K2.5-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+  - MODEL_NAME=Kimi-K2.5-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh

 - label: "Kimi-K2.6-MXFP4-PD-1P1D-TP8-MoRIIO-proxy"
  agents:
    queue: amd_mi350_ainic
-  timeout_in_minutes: 120
+  timeout_in_minutes: 180
  concurrency: 2
  concurrency_group: "amd-disagg/mi350/2node"
  commands:
-  - MODEL_NAME=Kimi-K2.6-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+  - MODEL_NAME=Kimi-K2.6-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh

 - label: "Kimi-K2.6-MXFP4-PD-1P1D-TP8-MoRIIO-vllm-router"
  agents:
    queue: amd_mi350_ainic
-  timeout_in_minutes: 120
+  timeout_in_minutes: 180
  concurrency: 2
  concurrency_group: "amd-disagg/mi350/2node"
  commands:
-  - MODEL_NAME=Kimi-K2.6-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+  - MODEL_NAME=Kimi-K2.6-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh

 - label: "MiniMax-M3-MXFP8-PD-1P1D-TP8-MoRIIO-proxy"
  agents:
    queue: amd_mi350_ainic
-  timeout_in_minutes: 120
+  timeout_in_minutes: 180
  concurrency: 2
  concurrency_group: "amd-disagg/mi350/2node"
  commands:
-  - MODEL_NAME=MiniMax-M3-MXFP8 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+  - MODEL_NAME=MiniMax-M3-MXFP8 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh

 - label: "MiniMax-M3-MXFP8-PD-1P1D-TP8-MoRIIO-vllm-router"
  agents:
    queue: amd_mi350_ainic
-  timeout_in_minutes: 120
+  timeout_in_minutes: 180
  concurrency: 2
  concurrency_group: "amd-disagg/mi350/2node"
  commands:
-  - MODEL_NAME=MiniMax-M3-MXFP8 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+  - MODEL_NAME=MiniMax-M3-MXFP8 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh

 #--------------------------------------  mi350 · disagg 2P2D (TP8, 4 nodes) · SLURM  --------------------------------------#

 - label: "DeepSeek-V3-PD-2P2D-TP8-MoRIIO-proxy"
  agents:
    queue: amd_mi350_ainic
-  timeout_in_minutes: 120
+  timeout_in_minutes: 180
  concurrency: 2
  concurrency_group: "amd-disagg/mi350/4node"
  commands:
-  - MODEL_NAME=DeepSeek-V3 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+  - MODEL_NAME=DeepSeek-V3 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh

 - label: "DeepSeek-V3-PD-2P2D-TP8-MoRIIO-vllm-router"
  agents:
    queue: amd_mi350_ainic
-  timeout_in_minutes: 120
+  timeout_in_minutes: 180
  concurrency: 2
  concurrency_group: "amd-disagg/mi350/4node"
  commands:
-  - MODEL_NAME=DeepSeek-V3 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+  - MODEL_NAME=DeepSeek-V3 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh

 - label: "DeepSeek-R1-MXFP4-PD-2P2D-TP8-MoRIIO-proxy"
  agents:
    queue: amd_mi350_ainic
-  timeout_in_minutes: 120
+  timeout_in_minutes: 180
  concurrency: 2
  concurrency_group: "amd-disagg/mi350/4node"
  commands:
-  - MODEL_NAME=DeepSeek-R1-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+  - MODEL_NAME=DeepSeek-R1-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh

 - label: "DeepSeek-R1-MXFP4-PD-2P2D-TP8-MoRIIO-vllm-router"
  agents:
    queue: amd_mi350_ainic
-  timeout_in_minutes: 120
+  timeout_in_minutes: 180
  concurrency: 2
  concurrency_group: "amd-disagg/mi350/4node"
  commands:
-  - MODEL_NAME=DeepSeek-R1-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+  - MODEL_NAME=DeepSeek-R1-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh

 - label: "Kimi-K2.5-MXFP4-PD-2P2D-TP8-MoRIIO-proxy"
  agents:
    queue: amd_mi350_ainic
-  timeout_in_minutes: 120
+  timeout_in_minutes: 180
  concurrency: 2
  concurrency_group: "amd-disagg/mi350/4node"
  commands:
-  - MODEL_NAME=Kimi-K2.5-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+  - MODEL_NAME=Kimi-K2.5-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh

 - label: "Kimi-K2.5-MXFP4-PD-2P2D-TP8-MoRIIO-vllm-router"
  agents:
    queue: amd_mi350_ainic
-  timeout_in_minutes: 120
+  timeout_in_minutes: 180
  concurrency: 2
  concurrency_group: "amd-disagg/mi350/4node"
  commands:
-  - MODEL_NAME=Kimi-K2.5-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+  - MODEL_NAME=Kimi-K2.5-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh

 - label: "Kimi-K2.6-MXFP4-PD-2P2D-TP8-MoRIIO-proxy"
  agents:
    queue: amd_mi350_ainic
-  timeout_in_minutes: 120
+  timeout_in_minutes: 180
  concurrency: 2
  concurrency_group: "amd-disagg/mi350/4node"
  commands:
-  - MODEL_NAME=Kimi-K2.6-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+  - MODEL_NAME=Kimi-K2.6-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh

 - label: "Kimi-K2.6-MXFP4-PD-2P2D-TP8-MoRIIO-vllm-router"
  agents:
    queue: amd_mi350_ainic
-  timeout_in_minutes: 120
+  timeout_in_minutes: 180
  concurrency: 2
  concurrency_group: "amd-disagg/mi350/4node"
  commands:
-  - MODEL_NAME=Kimi-K2.6-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+  - MODEL_NAME=Kimi-K2.6-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh

 - label: "MiniMax-M3-MXFP8-PD-2P2D-TP8-MoRIIO-proxy"
  agents:
    queue: amd_mi350_ainic
-  timeout_in_minutes: 120
+  timeout_in_minutes: 180
  concurrency: 2
  concurrency_group: "amd-disagg/mi350/4node"
  commands:
-  - MODEL_NAME=MiniMax-M3-MXFP8 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+  - MODEL_NAME=MiniMax-M3-MXFP8 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh

 - label: "MiniMax-M3-MXFP8-PD-2P2D-TP8-MoRIIO-vllm-router"
  agents:
    queue: amd_mi350_ainic
-  timeout_in_minutes: 120
+  timeout_in_minutes: 180
  concurrency: 2
  concurrency_group: "amd-disagg/mi350/4node"
  commands:
-  - MODEL_NAME=MiniMax-M3-MXFP8 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+  - MODEL_NAME=MiniMax-M3-MXFP8 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh

-#--------------------------------------  mi350 · disagg wide-EP (disabled)  --------------------------------------#
+#--------------------------------------  mi350 · disagg wide-EP intranode (EP8/DP8, 2 nodes) · SLURM  --------------------------------------#

-# - label: "DeepSeek-V3-PD-1P1D-EP8/DP8-WideEP-MoRIIO-proxy"
-#   agents:
-#     queue: amd_mi350_ainic
-#   timeout_in_minutes: 120
-#   concurrency: 2
-#   concurrency_group: "amd-disagg/mi350/2node"
-#   commands:
-#   - IMAGE=vllm/vllm-openai-rocm:nightly MODEL_NAME=DeepSeek-V3 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+- label: "DeepSeek-V3-PD-1P1D-EP8/DP8-WideEP-MoRIIO-proxy"
+  agents:
+    queue: amd_mi350_ainic
+  timeout_in_minutes: 180
+  concurrency: 2
+  concurrency_group: "amd-disagg/mi350/2node"
+  commands:
+  - MODEL_NAME=DeepSeek-V3 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+
+- label: "DeepSeek-V3-PD-1P1D-EP8/DP8-WideEP-MoRIIO-vllm-router"
+  agents:
+    queue: amd_mi350_ainic
+  timeout_in_minutes: 180
+  concurrency: 2
+  concurrency_group: "amd-disagg/mi350/2node"
+  commands:
+  - MODEL_NAME=DeepSeek-V3 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+
+- label: "DeepSeek-R1-MXFP4-PD-1P1D-EP8/DP8-WideEP-MoRIIO-proxy"
+  agents:
+    queue: amd_mi350_ainic
+  timeout_in_minutes: 180
+  concurrency: 2
+  concurrency_group: "amd-disagg/mi350/2node"
+  commands:
+  - MODEL_NAME=DeepSeek-R1-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+
+- label: "DeepSeek-R1-MXFP4-PD-1P1D-EP8/DP8-WideEP-MoRIIO-vllm-router"
+  agents:
+    queue: amd_mi350_ainic
+  timeout_in_minutes: 180
+  concurrency: 2
+  concurrency_group: "amd-disagg/mi350/2node"
+  commands:
+  - MODEL_NAME=DeepSeek-R1-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+
+- label: "Kimi-K2.5-MXFP4-PD-1P1D-EP8/DP8-WideEP-MoRIIO-proxy"
+  agents:
+    queue: amd_mi350_ainic
+  timeout_in_minutes: 180
+  concurrency: 2
+  concurrency_group: "amd-disagg/mi350/2node"
+  commands:
+  - MODEL_NAME=Kimi-K2.5-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+
+- label: "Kimi-K2.5-MXFP4-PD-1P1D-EP8/DP8-WideEP-MoRIIO-vllm-router"
+  agents:
+    queue: amd_mi350_ainic
+  timeout_in_minutes: 180
+  concurrency: 2
+  concurrency_group: "amd-disagg/mi350/2node"
+  commands:
+  - MODEL_NAME=Kimi-K2.5-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+
+- label: "Kimi-K2.6-MXFP4-PD-1P1D-EP8/DP8-WideEP-MoRIIO-proxy"
+  agents:
+    queue: amd_mi350_ainic
+  timeout_in_minutes: 180
+  concurrency: 2
+  concurrency_group: "amd-disagg/mi350/2node"
+  commands:
+  - MODEL_NAME=Kimi-K2.6-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+
+- label: "Kimi-K2.6-MXFP4-PD-1P1D-EP8/DP8-WideEP-MoRIIO-vllm-router"
+  agents:
+    queue: amd_mi350_ainic
+  timeout_in_minutes: 180
+  concurrency: 2
+  concurrency_group: "amd-disagg/mi350/2node"
+  commands:
+  - MODEL_NAME=Kimi-K2.6-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+
+- label: "MiniMax-M3-MXFP8-PD-1P1D-EP8/DP8-WideEP-MoRIIO-proxy"
+  agents:
+    queue: amd_mi350_ainic
+  timeout_in_minutes: 180
+  concurrency: 2
+  concurrency_group: "amd-disagg/mi350/2node"
+  commands:
+  - MODEL_NAME=MiniMax-M3-MXFP8 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
+
+- label: "MiniMax-M3-MXFP8-PD-1P1D-EP8/DP8-WideEP-MoRIIO-vllm-router"
+  agents:
+    queue: amd_mi350_ainic
+  timeout_in_minutes: 180
+  concurrency: 2
+  concurrency_group: "amd-disagg/mi350/2node"
+  commands:
+  - MODEL_NAME=MiniMax-M3-MXFP8 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh
@@ -37,12 +37,12 @@ IMAGE="${IMAGE:-vllm/vllm-openai-rocm:nightly}"
 NODES="${NODES:-2}"
 GPUS_PER_NODE="${GPUS_PER_NODE:-8}"
 PARTITION="${SLURM_PARTITION:-}"
-TIME_LIMIT="${SLURM_TIME_LIMIT:-02:00:00}"
+TIME_LIMIT="${SLURM_TIME_LIMIT:-03:00:00}"
 WIDE_EP_MODE="${WIDE_EP_MODE:-0}"              # 0 -> 1P1D TP8 (default); 1 -> wide-EP
 xP="${xP:-1}"
 yD="${yD:-1}"
 RUN_AFTER_HEALTH="${RUN_AFTER_HEALTH:-accuracy}"
-HEALTH_TIMEOUT_S="${HEALTH_TIMEOUT_S:-3600}"   # P/D bring-up budget; +900s grace must fit the 2h wall
+HEALTH_TIMEOUT_S="${HEALTH_TIMEOUT_S:-3600}"   # P/D bring-up budget; +900s grace must fit the CI step timeout
 SHARED_MOUNT="${SHARED_MOUNT:-/data}"
 LOG_ROOT="${LOG_ROOT:-${SHARED_MOUNT}/${USER:-$(whoami)}/disagg_logs}"
 DRY_RUN="${DRY_RUN:-0}"
@@ -4,7 +4,6 @@
 #SBATCH --ntasks-per-node=1
 #SBATCH --gres=gpu:8
 #SBATCH --exclusive
-#SBATCH --time=02:00:00
 #SBATCH --chdir=/tmp
 #SBATCH --output=/tmp/spur-%j.out
 #SBATCH --error=/tmp/spur-%j.err