From 28a2ccee78699e90ad06c7e833dc018dea8fddbf Mon Sep 17 00:00:00 2001 From: Chaitanya Sri Krishna Lolla Date: Wed, 9 Sep 2026 03:36:36 +0530 Subject: [PATCH] [ROCm][DI][CI] Enable WideEP Intranode tests (#53195) Signed-off-by: lcskrishna --- .buildkite/amd-disagg/models.yaml | 15 +- .buildkite/amd-disagg/pipeline-disagg.yaml | 179 +++++++++++++----- .../amd-disagg/run-slurm-disagg-test.sh | 4 +- .buildkite/amd-disagg/run_xPyD_disagg.slurm | 1 - 4 files changed, 144 insertions(+), 55 deletions(-) diff --git a/.buildkite/amd-disagg/models.yaml b/.buildkite/amd-disagg/models.yaml index e919475db8..028b2c0017 100644 --- a/.buildkite/amd-disagg/models.yaml +++ b/.buildkite/amd-disagg/models.yaml @@ -37,14 +37,14 @@ models: VLLM_ROCM_USE_AITER_MOE: "1" VLLM_ROCM_USE_AITER_RMSNORM: "1" VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: "0" + VLLM_ROCM_USE_AITER_FP8BMM: "false" base_flags: "--trust-remote-code --kv-cache-dtype fp8" prefill: tp: "--gpu-memory-utilization 0.85" ep: "--gpu-memory-utilization 0.85 --enforce-eager" decode: tp: "--gpu-memory-utilization 0.85" - #ep: '--gpu-memory-utilization 0.75 --enforce-eager' - ep: '--gpu-memory-utilization 0.75 --compilation-config {"cudagraph_mode":"PIECEWISE","custom_ops":["+quant_fp8"]}' + ep: '--gpu-memory-utilization 0.75 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8"]}' - model: MiniMax-M3-MXFP8 env: @@ -53,11 +53,14 @@ models: VLLM_ROCM_USE_AITER_RMSNORM: "1" VLLM_USE_BREAKABLE_CUDAGRAPH: "0" VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: "INT6" + VLLM_KV_CACHE_LAYOUT: "HND" base_flags: "--trust-remote-code --attention-backend TRITON_ATTN --block-size 128 --language-model-only --kv-cache-dtype fp8" prefill: tp: "--gpu-memory-utilization 0.85 --enforce-eager" + ep: "--gpu-memory-utilization 0.85 --enforce-eager" decode: tp: "--gpu-memory-utilization 0.85" + ep: '--gpu-memory-utilization 0.75 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8"]}' - model: DeepSeek-R1-MXFP4 env: @@ -65,11 +68,14 @@ models: VLLM_ROCM_USE_AITER_MLA: "1" VLLM_ROCM_USE_AITER_MOE: "1" VLLM_ROCM_USE_AITER_RMSNORM: "1" + VLLM_ROCM_USE_AITER_FP8BMM: "false" base_flags: "--trust-remote-code --kv-cache-dtype fp8" prefill: tp: "--gpu-memory-utilization 0.85" + ep: "--gpu-memory-utilization 0.85 --enforce-eager" decode: tp: "--gpu-memory-utilization 0.85" + ep: '--gpu-memory-utilization 0.75 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8"]}' - model: Kimi-K2.5-MXFP4 env: @@ -79,8 +85,10 @@ models: base_flags: "--trust-remote-code" prefill: tp: "--gpu-memory-utilization 0.85" + ep: "--gpu-memory-utilization 0.85 --enforce-eager" decode: tp: "--gpu-memory-utilization 0.85" + ep: '--gpu-memory-utilization 0.75 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8"]}' - model: Kimi-K2.6-MXFP4 env: @@ -89,9 +97,10 @@ models: VLLM_ROCM_USE_AITER_MLA: "1" VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: "INT4" VLLM_ROCM_USE_SKINNY_GEMM: "0" - VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: "1" base_flags: "--trust-remote-code --kv-cache-dtype fp8 --mm-encoder-tp-mode data --block-size 1 --attention-backend ROCM_AITER_MLA" prefill: tp: "--gpu-memory-utilization 0.9" + ep: "--gpu-memory-utilization 0.85 --enforce-eager" decode: tp: "--gpu-memory-utilization 0.9" + ep: '--gpu-memory-utilization 0.75 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8"]}' diff --git a/.buildkite/amd-disagg/pipeline-disagg.yaml b/.buildkite/amd-disagg/pipeline-disagg.yaml index cf8fe0d230..cd31314dfa 100644 --- a/.buildkite/amd-disagg/pipeline-disagg.yaml +++ b/.buildkite/amd-disagg/pipeline-disagg.yaml @@ -11,192 +11,273 @@ steps: - label: "DeepSeek-V3-PD-1P1D-TP8-MoRIIO-proxy" agents: queue: amd_mi350_ainic - timeout_in_minutes: 120 + timeout_in_minutes: 180 concurrency: 2 concurrency_group: "amd-disagg/mi350/2node" commands: - - MODEL_NAME=DeepSeek-V3 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + - MODEL_NAME=DeepSeek-V3 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - label: "DeepSeek-V3-PD-1P1D-TP8-MoRIIO-vllm-router" agents: queue: amd_mi350_ainic - timeout_in_minutes: 120 + timeout_in_minutes: 180 concurrency: 2 concurrency_group: "amd-disagg/mi350/2node" commands: - - MODEL_NAME=DeepSeek-V3 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + - MODEL_NAME=DeepSeek-V3 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - label: "DeepSeek-R1-MXFP4-PD-1P1D-TP8-MoRIIO-proxy" agents: queue: amd_mi350_ainic - timeout_in_minutes: 120 + timeout_in_minutes: 180 concurrency: 2 concurrency_group: "amd-disagg/mi350/2node" commands: - - MODEL_NAME=DeepSeek-R1-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + - MODEL_NAME=DeepSeek-R1-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - label: "DeepSeek-R1-MXFP4-PD-1P1D-TP8-MoRIIO-vllm-router" agents: queue: amd_mi350_ainic - timeout_in_minutes: 120 + timeout_in_minutes: 180 concurrency: 2 concurrency_group: "amd-disagg/mi350/2node" commands: - - MODEL_NAME=DeepSeek-R1-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + - MODEL_NAME=DeepSeek-R1-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - label: "Kimi-K2.5-MXFP4-PD-1P1D-TP8-MoRIIO-proxy" agents: queue: amd_mi350_ainic - timeout_in_minutes: 120 + timeout_in_minutes: 180 concurrency: 2 concurrency_group: "amd-disagg/mi350/2node" commands: - - MODEL_NAME=Kimi-K2.5-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + - MODEL_NAME=Kimi-K2.5-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - label: "Kimi-K2.5-MXFP4-PD-1P1D-TP8-MoRIIO-vllm-router" agents: queue: amd_mi350_ainic - timeout_in_minutes: 120 + timeout_in_minutes: 180 concurrency: 2 concurrency_group: "amd-disagg/mi350/2node" commands: - - MODEL_NAME=Kimi-K2.5-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + - MODEL_NAME=Kimi-K2.5-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - label: "Kimi-K2.6-MXFP4-PD-1P1D-TP8-MoRIIO-proxy" agents: queue: amd_mi350_ainic - timeout_in_minutes: 120 + timeout_in_minutes: 180 concurrency: 2 concurrency_group: "amd-disagg/mi350/2node" commands: - - MODEL_NAME=Kimi-K2.6-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + - MODEL_NAME=Kimi-K2.6-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - label: "Kimi-K2.6-MXFP4-PD-1P1D-TP8-MoRIIO-vllm-router" agents: queue: amd_mi350_ainic - timeout_in_minutes: 120 + timeout_in_minutes: 180 concurrency: 2 concurrency_group: "amd-disagg/mi350/2node" commands: - - MODEL_NAME=Kimi-K2.6-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + - MODEL_NAME=Kimi-K2.6-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - label: "MiniMax-M3-MXFP8-PD-1P1D-TP8-MoRIIO-proxy" agents: queue: amd_mi350_ainic - timeout_in_minutes: 120 + timeout_in_minutes: 180 concurrency: 2 concurrency_group: "amd-disagg/mi350/2node" commands: - - MODEL_NAME=MiniMax-M3-MXFP8 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + - MODEL_NAME=MiniMax-M3-MXFP8 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - label: "MiniMax-M3-MXFP8-PD-1P1D-TP8-MoRIIO-vllm-router" agents: queue: amd_mi350_ainic - timeout_in_minutes: 120 + timeout_in_minutes: 180 concurrency: 2 concurrency_group: "amd-disagg/mi350/2node" commands: - - MODEL_NAME=MiniMax-M3-MXFP8 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + - MODEL_NAME=MiniMax-M3-MXFP8 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh #-------------------------------------- mi350 · disagg 2P2D (TP8, 4 nodes) · SLURM --------------------------------------# - label: "DeepSeek-V3-PD-2P2D-TP8-MoRIIO-proxy" agents: queue: amd_mi350_ainic - timeout_in_minutes: 120 + timeout_in_minutes: 180 concurrency: 2 concurrency_group: "amd-disagg/mi350/4node" commands: - - MODEL_NAME=DeepSeek-V3 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + - MODEL_NAME=DeepSeek-V3 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - label: "DeepSeek-V3-PD-2P2D-TP8-MoRIIO-vllm-router" agents: queue: amd_mi350_ainic - timeout_in_minutes: 120 + timeout_in_minutes: 180 concurrency: 2 concurrency_group: "amd-disagg/mi350/4node" commands: - - MODEL_NAME=DeepSeek-V3 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + - MODEL_NAME=DeepSeek-V3 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - label: "DeepSeek-R1-MXFP4-PD-2P2D-TP8-MoRIIO-proxy" agents: queue: amd_mi350_ainic - timeout_in_minutes: 120 + timeout_in_minutes: 180 concurrency: 2 concurrency_group: "amd-disagg/mi350/4node" commands: - - MODEL_NAME=DeepSeek-R1-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + - MODEL_NAME=DeepSeek-R1-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - label: "DeepSeek-R1-MXFP4-PD-2P2D-TP8-MoRIIO-vllm-router" agents: queue: amd_mi350_ainic - timeout_in_minutes: 120 + timeout_in_minutes: 180 concurrency: 2 concurrency_group: "amd-disagg/mi350/4node" commands: - - MODEL_NAME=DeepSeek-R1-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + - MODEL_NAME=DeepSeek-R1-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - label: "Kimi-K2.5-MXFP4-PD-2P2D-TP8-MoRIIO-proxy" agents: queue: amd_mi350_ainic - timeout_in_minutes: 120 + timeout_in_minutes: 180 concurrency: 2 concurrency_group: "amd-disagg/mi350/4node" commands: - - MODEL_NAME=Kimi-K2.5-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + - MODEL_NAME=Kimi-K2.5-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - label: "Kimi-K2.5-MXFP4-PD-2P2D-TP8-MoRIIO-vllm-router" agents: queue: amd_mi350_ainic - timeout_in_minutes: 120 + timeout_in_minutes: 180 concurrency: 2 concurrency_group: "amd-disagg/mi350/4node" commands: - - MODEL_NAME=Kimi-K2.5-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + - MODEL_NAME=Kimi-K2.5-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - label: "Kimi-K2.6-MXFP4-PD-2P2D-TP8-MoRIIO-proxy" agents: queue: amd_mi350_ainic - timeout_in_minutes: 120 + timeout_in_minutes: 180 concurrency: 2 concurrency_group: "amd-disagg/mi350/4node" commands: - - MODEL_NAME=Kimi-K2.6-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + - MODEL_NAME=Kimi-K2.6-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - label: "Kimi-K2.6-MXFP4-PD-2P2D-TP8-MoRIIO-vllm-router" agents: queue: amd_mi350_ainic - timeout_in_minutes: 120 + timeout_in_minutes: 180 concurrency: 2 concurrency_group: "amd-disagg/mi350/4node" commands: - - MODEL_NAME=Kimi-K2.6-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + - MODEL_NAME=Kimi-K2.6-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - label: "MiniMax-M3-MXFP8-PD-2P2D-TP8-MoRIIO-proxy" agents: queue: amd_mi350_ainic - timeout_in_minutes: 120 + timeout_in_minutes: 180 concurrency: 2 concurrency_group: "amd-disagg/mi350/4node" commands: - - MODEL_NAME=MiniMax-M3-MXFP8 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + - MODEL_NAME=MiniMax-M3-MXFP8 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - label: "MiniMax-M3-MXFP8-PD-2P2D-TP8-MoRIIO-vllm-router" agents: queue: amd_mi350_ainic - timeout_in_minutes: 120 + timeout_in_minutes: 180 concurrency: 2 concurrency_group: "amd-disagg/mi350/4node" commands: - - MODEL_NAME=MiniMax-M3-MXFP8 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + - MODEL_NAME=MiniMax-M3-MXFP8 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh -#-------------------------------------- mi350 · disagg wide-EP (disabled) --------------------------------------# +#-------------------------------------- mi350 · disagg wide-EP intranode (EP8/DP8, 2 nodes) · SLURM --------------------------------------# -# - label: "DeepSeek-V3-PD-1P1D-EP8/DP8-WideEP-MoRIIO-proxy" -# agents: -# queue: amd_mi350_ainic -# timeout_in_minutes: 120 -# concurrency: 2 -# concurrency_group: "amd-disagg/mi350/2node" -# commands: -# - IMAGE=vllm/vllm-openai-rocm:nightly MODEL_NAME=DeepSeek-V3 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 SLURM_TIME_LIMIT=02:00:00 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh +- label: "DeepSeek-V3-PD-1P1D-EP8/DP8-WideEP-MoRIIO-proxy" + agents: + queue: amd_mi350_ainic + timeout_in_minutes: 180 + concurrency: 2 + concurrency_group: "amd-disagg/mi350/2node" + commands: + - MODEL_NAME=DeepSeek-V3 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + +- label: "DeepSeek-V3-PD-1P1D-EP8/DP8-WideEP-MoRIIO-vllm-router" + agents: + queue: amd_mi350_ainic + timeout_in_minutes: 180 + concurrency: 2 + concurrency_group: "amd-disagg/mi350/2node" + commands: + - MODEL_NAME=DeepSeek-V3 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + +- label: "DeepSeek-R1-MXFP4-PD-1P1D-EP8/DP8-WideEP-MoRIIO-proxy" + agents: + queue: amd_mi350_ainic + timeout_in_minutes: 180 + concurrency: 2 + concurrency_group: "amd-disagg/mi350/2node" + commands: + - MODEL_NAME=DeepSeek-R1-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + +- label: "DeepSeek-R1-MXFP4-PD-1P1D-EP8/DP8-WideEP-MoRIIO-vllm-router" + agents: + queue: amd_mi350_ainic + timeout_in_minutes: 180 + concurrency: 2 + concurrency_group: "amd-disagg/mi350/2node" + commands: + - MODEL_NAME=DeepSeek-R1-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + +- label: "Kimi-K2.5-MXFP4-PD-1P1D-EP8/DP8-WideEP-MoRIIO-proxy" + agents: + queue: amd_mi350_ainic + timeout_in_minutes: 180 + concurrency: 2 + concurrency_group: "amd-disagg/mi350/2node" + commands: + - MODEL_NAME=Kimi-K2.5-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + +- label: "Kimi-K2.5-MXFP4-PD-1P1D-EP8/DP8-WideEP-MoRIIO-vllm-router" + agents: + queue: amd_mi350_ainic + timeout_in_minutes: 180 + concurrency: 2 + concurrency_group: "amd-disagg/mi350/2node" + commands: + - MODEL_NAME=Kimi-K2.5-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + +- label: "Kimi-K2.6-MXFP4-PD-1P1D-EP8/DP8-WideEP-MoRIIO-proxy" + agents: + queue: amd_mi350_ainic + timeout_in_minutes: 180 + concurrency: 2 + concurrency_group: "amd-disagg/mi350/2node" + commands: + - MODEL_NAME=Kimi-K2.6-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + +- label: "Kimi-K2.6-MXFP4-PD-1P1D-EP8/DP8-WideEP-MoRIIO-vllm-router" + agents: + queue: amd_mi350_ainic + timeout_in_minutes: 180 + concurrency: 2 + concurrency_group: "amd-disagg/mi350/2node" + commands: + - MODEL_NAME=Kimi-K2.6-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + +- label: "MiniMax-M3-MXFP8-PD-1P1D-EP8/DP8-WideEP-MoRIIO-proxy" + agents: + queue: amd_mi350_ainic + timeout_in_minutes: 180 + concurrency: 2 + concurrency_group: "amd-disagg/mi350/2node" + commands: + - MODEL_NAME=MiniMax-M3-MXFP8 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh + +- label: "MiniMax-M3-MXFP8-PD-1P1D-EP8/DP8-WideEP-MoRIIO-vllm-router" + agents: + queue: amd_mi350_ainic + timeout_in_minutes: 180 + concurrency: 2 + concurrency_group: "amd-disagg/mi350/2node" + commands: + - MODEL_NAME=MiniMax-M3-MXFP8 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh diff --git a/.buildkite/amd-disagg/run-slurm-disagg-test.sh b/.buildkite/amd-disagg/run-slurm-disagg-test.sh index e4beeb577f..6e062ec338 100644 --- a/.buildkite/amd-disagg/run-slurm-disagg-test.sh +++ b/.buildkite/amd-disagg/run-slurm-disagg-test.sh @@ -37,12 +37,12 @@ IMAGE="${IMAGE:-vllm/vllm-openai-rocm:nightly}" NODES="${NODES:-2}" GPUS_PER_NODE="${GPUS_PER_NODE:-8}" PARTITION="${SLURM_PARTITION:-}" -TIME_LIMIT="${SLURM_TIME_LIMIT:-02:00:00}" +TIME_LIMIT="${SLURM_TIME_LIMIT:-03:00:00}" WIDE_EP_MODE="${WIDE_EP_MODE:-0}" # 0 -> 1P1D TP8 (default); 1 -> wide-EP xP="${xP:-1}" yD="${yD:-1}" RUN_AFTER_HEALTH="${RUN_AFTER_HEALTH:-accuracy}" -HEALTH_TIMEOUT_S="${HEALTH_TIMEOUT_S:-3600}" # P/D bring-up budget; +900s grace must fit the 2h wall +HEALTH_TIMEOUT_S="${HEALTH_TIMEOUT_S:-3600}" # P/D bring-up budget; +900s grace must fit the CI step timeout SHARED_MOUNT="${SHARED_MOUNT:-/data}" LOG_ROOT="${LOG_ROOT:-${SHARED_MOUNT}/${USER:-$(whoami)}/disagg_logs}" DRY_RUN="${DRY_RUN:-0}" diff --git a/.buildkite/amd-disagg/run_xPyD_disagg.slurm b/.buildkite/amd-disagg/run_xPyD_disagg.slurm index de50b40f52..c5513b699e 100644 --- a/.buildkite/amd-disagg/run_xPyD_disagg.slurm +++ b/.buildkite/amd-disagg/run_xPyD_disagg.slurm @@ -4,7 +4,6 @@ #SBATCH --ntasks-per-node=1 #SBATCH --gres=gpu:8 #SBATCH --exclusive -#SBATCH --time=02:00:00 #SBATCH --chdir=/tmp #SBATCH --output=/tmp/spur-%j.out #SBATCH --error=/tmp/spur-%j.err