diff --git a/.buildkite/.pipeline_gen_v2 b/.buildkite/.pipeline_gen_v2 deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/.buildkite/amd-disagg/cluster.sh b/.buildkite/amd-disagg/cluster.sh deleted file mode 100644 index 4b377dca08..0000000000 --- a/.buildkite/amd-disagg/cluster.sh +++ /dev/null @@ -1,166 +0,0 @@ -#!/bin/bash -# ============================================================================= -# cluster.sh — single, generic config for the vLLM disaggregated (P/D) launcher. -# ----------------------------------------------------------------------------- -# This script is *sourced* by vllm_disagg.sh. -# -# Every value is `${VAR:-default}`, so the environment always wins: -# environment variable > built-in default below -# -# So you can override anything inline: -# PREFILL_IP=10.0.0.1 DECODE_IP=10.0.0.2 ./vllm_disagg.sh prefill -# -# Site-specific values (model dir, IPs, NIC list, partition) are the defaults -# in the "site defaults" sections below — edit those for a new cluster. -# Model-SPECIFIC perf flags live in models.yaml, NOT here. -# ============================================================================= - -_CLUSTER_SH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -# ----------------------------------------------------------------- model / mode -# MODEL_NAME indexes into models.yaml; MODEL_DIR is the parent dir holding it. -# MODEL_PATH (resolved by the launcher) = ${MODEL_DIR}/${MODEL_NAME}. [site] -export MODEL_NAME="${MODEL_NAME:-DeepSeek-V3}" -export MODEL_DIR="${MODEL_DIR:-/data/models2}" - -# Shared NFS root (5 TB, visible on every node): model weights + per-run logs. [site] -export SHARED_MOUNT="${SHARED_MOUNT:-/data}" - -# Parallelism mode (launcher derives PARALLEL_MODE tp|ep from this): -# WIDE_EP_MODE=0 tp : each node is an independent TP server (TP8, 1P1D) -# WIDE_EP_MODE=1 ep : data-parallel + expert-parallel across xP/yD nodes (wideep) -export WIDE_EP_MODE="${WIDE_EP_MODE:-0}" - -# ----------------------------------------------------------------- topology -# xP prefill nodes + yD decode nodes. IPADDRS is the ordered, comma-separated -# node list (prefill IPs first, then decode IPs). NODE_RANK is this node's global -# 0-based rank; under SLURM it defaults to $SLURM_PROCID. Leave IPADDRS empty to -# use the PREFILL_IP/DECODE_IP fallback defaults below (1P1D only). -export xP="${xP:-1}" -export yD="${yD:-1}" -export IPADDRS="${IPADDRS:-}" -export NODE_RANK="${NODE_RANK:-${SLURM_PROCID:-}}" -export PREFILL_IP="${PREFILL_IP:-10.0.0.1}" -export DECODE_IP="${DECODE_IP:-10.0.0.2}" - -# Per-node GPU count and TP degree -export GPUS_PER_NODE="${GPUS_PER_NODE:-8}" -export TP_SIZE="${TP_SIZE:-${GPUS_PER_NODE}}" - -# ----------------------------------------------------------------- ports -# TP-mode (WIDE_EP_MODE=0) server ports. -export PREFILL_PORT="${PREFILL_PORT:-8100}" -export DECODE_PORT="${DECODE_PORT:-8200}" - -# EP-mode (WIDE_EP_MODE=1) ports: API serve port, DP RPC port, KV transfer port, and -# per-node MoRIIO local ping port (must differ from PROXY_PING_PORT). -export SERVE_PORT="${SERVE_PORT:-20005}" -export RPC_PORT="${RPC_PORT:-13345}" -export KV_PORT="${KV_PORT:-9711}" -export LOCAL_PING_PORT="${LOCAL_PING_PORT:-61555}" - -# MoRIIO proxy: HTTP port clients/benchmark hit, plus the connector control ports. -# PROXY_PING_PORT MUST be 36367 — the proxy hardcodes its zmq service-discovery -# socket on that port; prefill/decode register to PROXY_IP:PROXY_PING_PORT. -export PROXY_IP="${PROXY_IP:-${PREFILL_IP}}" -export PROXY_PORT="${PROXY_PORT:-10001}" -export PROXY_PING_PORT="${PROXY_PING_PORT:-36367}" -export HANDSHAKE_PORT="${HANDSHAKE_PORT:-6301}" -export NOTIFY_PORT="${NOTIFY_PORT:-61005}" - -export PROXY_SCRIPT="${PROXY_SCRIPT:-/app/vllm/examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py}" - -# MoRIIO KV transfer direction (injected into --kv-transfer-config by the launcher): -# 0 -> omit read_mode (default; MoRIIO write mode: prefill pushes to decode) -# 1 -> "read_mode": true (decode pulls KV from prefill; matches upstream disagg) -export MORIIO_READ_MODE="${MORIIO_READ_MODE:-0}" - -# ----------------------------------------------------------------- router / gateway -# Selection for client (bench/accuracy) traffic: -# proxy -> the in-container MoRIIO proxy started by the launcher -# vllm-router -> an external `vllm/vllm-router` container started by the SLURM job -# on the rank-0 node (default) -# Both use the SAME MoRIIO discovery mechanism (prefill/decode register to -# PROXY_IP:PROXY_PING_PORT=36367); only the client HTTP front door differs. -export ROUTER_TYPE="${ROUTER_TYPE:-vllm-router}" -export ROUTER_PORT="${ROUTER_PORT:-30000}" -export ROUTER_POLICY="${ROUTER_POLICY:-round_robin}" -export VLLM_ROUTER_IMAGE="${VLLM_ROUTER_IMAGE:-vllm/vllm-router:nightly}" -# Single client-facing port bench/accuracy target: the router port when routing, -# else the proxy port. Env override always wins. -if [[ "${ROUTER_TYPE}" == "vllm-router" ]]; then - export GATEWAY_PORT="${GATEWAY_PORT:-${ROUTER_PORT}}" -else - export GATEWAY_PORT="${GATEWAY_PORT:-${PROXY_PORT}}" -fi - -# Where per-run logs / benchmark results are written. A $SLURM_JOB_ID subdir is -# appended so each CI run is self-scoped (falls back to 'local' off-SLURM). [site] -_LOG_BASE="${LOG_BASE:-/data/${USER:-$(id -un)}/disagg_logs}" -export LOG_PATH="${LOG_PATH:-${_LOG_BASE}/${SLURM_JOB_ID:-local}}" - -# ----------------------------------------------------------------- vLLM runtime -# Engine/platform/transport-level env (NOT model-specific). Model-architecture -# AITER kernel toggles live in models.yaml under each model's `env:` block. -export VLLM_USE_V1="${VLLM_USE_V1:-1}" -export HSA_NO_SCRATCH_RECLAIM="${HSA_NO_SCRATCH_RECLAIM:-1}" - -#export HF_HUB_OFFLINE="${HF_HUB_OFFLINE:-1}" -#export TRANSFORMERS_OFFLINE="${TRANSFORMERS_OFFLINE:-1}" -# -#export HOME=/tmp -export HF_HOME="${HF_HOME:-/tmp/hf_home}" -export XDG_CACHE_HOME="${XDG_CACHE_HOME:-/tmp/.cache}" -export VLLM_ENGINE_READY_TIMEOUT_S="${VLLM_ENGINE_READY_TIMEOUT_S:-3600}" - -# ----------------------------------------------------------------- scale -# DP/EP group formation timeout across nodes (seconds). -export DISTRIBUTED_TIMEOUT_SECONDS="${DISTRIBUTED_TIMEOUT_SECONDS:-7200}" - -# ----------------------------------------------------------------- RDMA / NCCL -# AMD Pensando AINIC RoCE fabric: 8 NICs exposed as ionic_0..7 (netdevs eth2..9), -# each rail on its own /24. GID index 1 + traffic class 104 -_IB_DEVICES="${IB_DEVICES:-ionic_0,ionic_1,ionic_2,ionic_3,ionic_4,ionic_5,ionic_6,ionic_7}" -_IB_GID_INDEX="${NCCL_IB_GID_INDEX:-1}" -export IB_DEVICES="${IB_DEVICES:-${_IB_DEVICES}}" -export NCCL_IB_HCA="${NCCL_IB_HCA:-${_IB_DEVICES}}" -export NCCL_IB_GID_INDEX="${NCCL_IB_GID_INDEX:-${_IB_GID_INDEX}}" -export NCCL_IB_DISABLE="${NCCL_IB_DISABLE:-0}" -# In-box RCCL net transport (no external plugin); pin bootstrap to the VPC iface. -export NCCL_NET_PLUGIN="${NCCL_NET_PLUGIN:-none}" -export NCCL_SOCKET_IFNAME="${NCCL_SOCKET_IFNAME:-eth0}" -export GLOO_SOCKET_IFNAME="${GLOO_SOCKET_IFNAME:-${NCCL_SOCKET_IFNAME}}" -export NCCL_CROSS_NIC="${NCCL_CROSS_NIC:-0}" -export NCCL_PXN_DISABLE="${NCCL_PXN_DISABLE:-0}" -export NCCL_NET_DISABLE_INTRA="${NCCL_NET_DISABLE_INTRA:-1}" -export NCCL_IB_TC="${NCCL_IB_TC:-104}" -export NCCL_IB_FIFO_TC="${NCCL_IB_FIFO_TC:-192}" -export NCCL_IB_QPS_PER_CONNECTION="${NCCL_IB_QPS_PER_CONNECTION:-1}" -export NCCL_IB_TIMEOUT="${NCCL_IB_TIMEOUT:-22}" -export NCCL_IB_RETRY_CNT="${NCCL_IB_RETRY_CNT:-12}" - -# MoRI uses the same NIC set as NCCL. -export MORI_RDMA_DEVICES="${MORI_RDMA_DEVICES:-${_IB_DEVICES}}" -export MORI_IB_GID_INDEX="${MORI_IB_GID_INDEX:-${_IB_GID_INDEX}}" -export MORI_SHMEM_HEAP_SIZE="${MORI_SHMEM_HEAP_SIZE:-16G}" - -# Pin to gfx950 to avoid jit compilation failures with other archs on this cluster. -export MORI_GPU_ARCHS="gfx950" - -# ----------------------------------------------------------------- benchmark -export BENCHMARK_COMBINATIONS="${BENCHMARK_COMBINATIONS:-1024/128 2048/128}" -export BENCHMARK_CON="${BENCHMARK_CON:-32 64}" -export NUM_PROMPTS_FACTOR="${NUM_PROMPTS_FACTOR:-2}" -export BENCHMARK_MIN_PROMPTS="${BENCHMARK_MIN_PROMPTS:-32}" - -# ----------------------------------------------------------------- accuracy -# `accuracy` role runs lm_eval (local-completions backend) against the proxy. -export ACCURACY_TASKS="${ACCURACY_TASKS:-gsm8k}" -export ACCURACY_NUM_CONCURRENT="${ACCURACY_NUM_CONCURRENT:-64}" -export ACCURACY_MAX_RETRIES="${ACCURACY_MAX_RETRIES:-3}" -export ACCURACY_METRIC="${ACCURACY_METRIC:-exact_match}" -export ACCURACY_THRESHOLD="${ACCURACY_THRESHOLD:-0.90}" - -# ----------------------------------------------------------------- SLURM (submit) -# Used by run-slurm-disagg-test.sh on the login node (harmless to export here). [site] -export SLURM_PARTITION="${SLURM_PARTITION:-default}" diff --git a/.buildkite/amd-disagg/models.yaml b/.buildkite/amd-disagg/models.yaml deleted file mode 100644 index 3744779528..0000000000 --- a/.buildkite/amd-disagg/models.yaml +++ /dev/null @@ -1,121 +0,0 @@ -# ============================================================================= -# Model catalog for the vLLM disaggregated (P/D) inference CI. -# ----------------------------------------------------------------------------- -# Loaded by vllm_disagg.sh for the selected MODEL_NAME. WIDE_EP_MODE selects the -# flag set (0 -> tp keys, 1 -> ep keys): -# tp -> base_flags + prefill.tp / decode.tp -# ep -> base_flags + prefill.ep / decode.ep -# -# Keep ONLY model-specific perf flags here. The launcher owns everything -# topological / mode-structural: -# - tp: --host/--port, --tensor-parallel-size, --kv-transfer-config -# - ep: --tp 1, --data-parallel-size[-local], --data-parallel-address/-rpc-port, -# --data-parallel-start-rank/--headless (children), --enable-expert-parallel, -# --all2all-backend mori, --no-enable-prefix-caching, -# --api-server-count + --kv-transfer-config (masters only) -# Do NOT put any of those in this file. -# -# Format: a top-level `models:` list. The launcher selects the entry whose -# `model:` matches MODEL_NAME. Each entry: -# model : name (must match MODEL_NAME; also the dir under MODEL_DIR) -# env : model/arch-specific environment variables. The launcher -# exports these BEFORE `vllm serve`, but only if they are -# not already set, so precedence is: -# caller/inline env > models.yaml env: -# (Transport/fabric/engine env stays in cluster.sh, -# e.g. MORI_RDMA_DEVICES, NCCL_IB_HCA, VLLM_USE_V1.) -# base_flags : always applied (both roles, both modes) -# prefill/decode. : role + mode specific perf flags -# experimental_flags : optional extra flags (omit if empty) -# ============================================================================= - -models: - - model: DeepSeek-V3 - env: - VLLM_ROCM_USE_AITER: "1" - VLLM_ROCM_USE_AITER_MLA: "1" - VLLM_ROCM_USE_AITER_MOE: "1" - VLLM_ROCM_USE_AITER_RMSNORM: "1" - VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: "0" - VLLM_ROCM_USE_AITER_FP8BMM: "false" - base_flags: "--trust-remote-code --kv-cache-dtype fp8" - prefill: - tp: "--gpu-memory-utilization 0.85" - ep: "--gpu-memory-utilization 0.85 --enforce-eager" - decode: - tp: "--gpu-memory-utilization 0.85" - ep: '--gpu-memory-utilization 0.75 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8"]}' - - - model: MiniMax-M3-MXFP8 - env: - VLLM_ROCM_USE_AITER: "1" - VLLM_ROCM_USE_AITER_MOE: "1" - VLLM_ROCM_USE_AITER_RMSNORM: "1" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: "INT6" - VLLM_KV_CACHE_LAYOUT: "HND" - base_flags: "--trust-remote-code --attention-backend TRITON_ATTN --block-size 128 --language-model-only --kv-cache-dtype fp8" - prefill: - tp: "--gpu-memory-utilization 0.85 --enforce-eager" - ep: "--gpu-memory-utilization 0.85 --enforce-eager" - decode: - tp: "--gpu-memory-utilization 0.85" - ep: '--gpu-memory-utilization 0.75 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8"]}' - - - model: DeepSeek-R1-MXFP4 - env: - VLLM_ROCM_USE_AITER: "1" - VLLM_ROCM_USE_AITER_MLA: "1" - VLLM_ROCM_USE_AITER_MOE: "1" - VLLM_ROCM_USE_AITER_RMSNORM: "1" - VLLM_ROCM_USE_AITER_FP8BMM: "false" - base_flags: "--trust-remote-code --kv-cache-dtype fp8" - prefill: - tp: "--gpu-memory-utilization 0.85" - ep: "--gpu-memory-utilization 0.85 --enforce-eager" - decode: - tp: "--gpu-memory-utilization 0.85" - ep: '--gpu-memory-utilization 0.75 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8"]}' - - - model: Kimi-K2.5-MXFP4 - env: - VLLM_ROCM_USE_AITER: "1" - VLLM_ROCM_USE_AITER_MOE: "1" - VLLM_ROCM_USE_AITER_RMSNORM: "1" - base_flags: "--trust-remote-code" - prefill: - tp: "--gpu-memory-utilization 0.85" - ep: "--gpu-memory-utilization 0.85 --enforce-eager" - decode: - tp: "--gpu-memory-utilization 0.85" - ep: '--gpu-memory-utilization 0.75 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8"]}' - - - model: Kimi-K2.6-MXFP4 - env: - VLLM_ROCM_USE_AITER: "1" - AMDGCN_USE_BUFFER_OPS: "1" - VLLM_ROCM_USE_AITER_MLA: "1" - VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: "INT4" - VLLM_ROCM_USE_SKINNY_GEMM: "0" - base_flags: "--trust-remote-code --kv-cache-dtype fp8 --mm-encoder-tp-mode data --block-size 1 --attention-backend ROCM_AITER_MLA" - prefill: - tp: "--gpu-memory-utilization 0.9" - ep: "--gpu-memory-utilization 0.85 --enforce-eager" - decode: - tp: "--gpu-memory-utilization 0.9" - ep: '--gpu-memory-utilization 0.75 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8"]}' - - - model: GLM-5.2-FP8 - env: - VLLM_ROCM_USE_AITER: "1" - VLLM_ROCM_USE_AITER_MLA: "1" - VLLM_ROCM_USE_AITER_MOE: "1" - VLLM_ROCM_USE_AITER_RMSNORM: "1" - VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: "0" - base_flags: "--trust-remote-code --kv-cache-dtype fp8" - prefill: - tp: "--gpu-memory-utilization 0.85" - ep: "--gpu-memory-utilization 0.85 --enforce-eager" - decode: - tp: "--gpu-memory-utilization 0.85" - ep: '--gpu-memory-utilization 0.85 --compilation-config {"cudagraph_mode":"PIECEWISE"}' diff --git a/.buildkite/amd-disagg/pipeline-disagg.yaml b/.buildkite/amd-disagg/pipeline-disagg.yaml deleted file mode 100644 index bf17770c1d..0000000000 --- a/.buildkite/amd-disagg/pipeline-disagg.yaml +++ /dev/null @@ -1,319 +0,0 @@ -# Native Buildkite pipeline — disaggregated PD SLURM tests (MoRIIO). -# -# ROUTER_TYPE must be set explicitly on every step: run-slurm-disagg-test.sh -# defaults to `vllm-router` -# - -steps: - -#-------------------------------------- mi350 · disagg 1P1D (TP8, 2 nodes) · SLURM --------------------------------------# - -- label: "DeepSeek-V3-PD-1P1D-TP8-MoRIIO-proxy" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/2node" - commands: - - MODEL_NAME=DeepSeek-V3 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "DeepSeek-V3-PD-1P1D-TP8-MoRIIO-vllm-router" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/2node" - commands: - - MODEL_NAME=DeepSeek-V3 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "DeepSeek-R1-MXFP4-PD-1P1D-TP8-MoRIIO-proxy" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/2node" - commands: - - MODEL_NAME=DeepSeek-R1-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "DeepSeek-R1-MXFP4-PD-1P1D-TP8-MoRIIO-vllm-router" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/2node" - commands: - - MODEL_NAME=DeepSeek-R1-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "Kimi-K2.5-MXFP4-PD-1P1D-TP8-MoRIIO-proxy" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/2node" - commands: - - MODEL_NAME=Kimi-K2.5-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "Kimi-K2.5-MXFP4-PD-1P1D-TP8-MoRIIO-vllm-router" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/2node" - commands: - - MODEL_NAME=Kimi-K2.5-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "Kimi-K2.6-MXFP4-PD-1P1D-TP8-MoRIIO-proxy" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/2node" - commands: - - MODEL_NAME=Kimi-K2.6-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "Kimi-K2.6-MXFP4-PD-1P1D-TP8-MoRIIO-vllm-router" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/2node" - commands: - - MODEL_NAME=Kimi-K2.6-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "MiniMax-M3-MXFP8-PD-1P1D-TP8-MoRIIO-proxy" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/2node" - commands: - - MODEL_NAME=MiniMax-M3-MXFP8 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "MiniMax-M3-MXFP8-PD-1P1D-TP8-MoRIIO-vllm-router" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/2node" - commands: - - MODEL_NAME=MiniMax-M3-MXFP8 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "GLM-5.2-FP8-PD-1P1D-TP8-MoRIIO-proxy" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/2node" - commands: - - MODEL_NAME=GLM-5.2-FP8 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "GLM-5.2-FP8-PD-1P1D-TP8-MoRIIO-vllm-router" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/2node" - commands: - - MODEL_NAME=GLM-5.2-FP8 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -#-------------------------------------- mi350 · disagg 2P2D (TP8, 4 nodes) · SLURM --------------------------------------# - -- label: "DeepSeek-V3-PD-2P2D-TP8-MoRIIO-proxy" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/4node" - commands: - - MODEL_NAME=DeepSeek-V3 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "DeepSeek-V3-PD-2P2D-TP8-MoRIIO-vllm-router" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/4node" - commands: - - MODEL_NAME=DeepSeek-V3 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "DeepSeek-R1-MXFP4-PD-2P2D-TP8-MoRIIO-proxy" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/4node" - commands: - - MODEL_NAME=DeepSeek-R1-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "DeepSeek-R1-MXFP4-PD-2P2D-TP8-MoRIIO-vllm-router" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/4node" - commands: - - MODEL_NAME=DeepSeek-R1-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "Kimi-K2.5-MXFP4-PD-2P2D-TP8-MoRIIO-proxy" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/4node" - commands: - - MODEL_NAME=Kimi-K2.5-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "Kimi-K2.5-MXFP4-PD-2P2D-TP8-MoRIIO-vllm-router" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/4node" - commands: - - MODEL_NAME=Kimi-K2.5-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "Kimi-K2.6-MXFP4-PD-2P2D-TP8-MoRIIO-proxy" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/4node" - commands: - - MODEL_NAME=Kimi-K2.6-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "Kimi-K2.6-MXFP4-PD-2P2D-TP8-MoRIIO-vllm-router" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/4node" - commands: - - MODEL_NAME=Kimi-K2.6-MXFP4 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "MiniMax-M3-MXFP8-PD-2P2D-TP8-MoRIIO-proxy" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/4node" - commands: - - MODEL_NAME=MiniMax-M3-MXFP8 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "MiniMax-M3-MXFP8-PD-2P2D-TP8-MoRIIO-vllm-router" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/4node" - commands: - - MODEL_NAME=MiniMax-M3-MXFP8 xP=2 yD=2 NODES=4 GPUS_PER_NODE=8 WIDE_EP_MODE=0 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -#-------------------------------------- mi350 · disagg wide-EP intranode (EP8/DP8, 2 nodes) · SLURM --------------------------------------# - -- label: "DeepSeek-V3-PD-1P1D-EP8/DP8-WideEP-MoRIIO-proxy" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/2node" - commands: - - MODEL_NAME=DeepSeek-V3 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "DeepSeek-V3-PD-1P1D-EP8/DP8-WideEP-MoRIIO-vllm-router" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/2node" - commands: - - MODEL_NAME=DeepSeek-V3 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "DeepSeek-R1-MXFP4-PD-1P1D-EP8/DP8-WideEP-MoRIIO-proxy" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/2node" - commands: - - MODEL_NAME=DeepSeek-R1-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "DeepSeek-R1-MXFP4-PD-1P1D-EP8/DP8-WideEP-MoRIIO-vllm-router" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/2node" - commands: - - MODEL_NAME=DeepSeek-R1-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "Kimi-K2.5-MXFP4-PD-1P1D-EP8/DP8-WideEP-MoRIIO-proxy" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/2node" - commands: - - MODEL_NAME=Kimi-K2.5-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "Kimi-K2.5-MXFP4-PD-1P1D-EP8/DP8-WideEP-MoRIIO-vllm-router" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/2node" - commands: - - MODEL_NAME=Kimi-K2.5-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "Kimi-K2.6-MXFP4-PD-1P1D-EP8/DP8-WideEP-MoRIIO-proxy" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/2node" - commands: - - MODEL_NAME=Kimi-K2.6-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "Kimi-K2.6-MXFP4-PD-1P1D-EP8/DP8-WideEP-MoRIIO-vllm-router" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/2node" - commands: - - MODEL_NAME=Kimi-K2.6-MXFP4 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "MiniMax-M3-MXFP8-PD-1P1D-EP8/DP8-WideEP-MoRIIO-proxy" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/2node" - commands: - - MODEL_NAME=MiniMax-M3-MXFP8 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "MiniMax-M3-MXFP8-PD-1P1D-EP8/DP8-WideEP-MoRIIO-vllm-router" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/2node" - commands: - - MODEL_NAME=MiniMax-M3-MXFP8 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "GLM-5.2-FP8-PD-1P1D-EP8/DP8-WideEP-MoRIIO-proxy" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/2node" - commands: - - MODEL_NAME=GLM-5.2-FP8 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=proxy WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh - -- label: "GLM-5.2-FP8-PD-1P1D-EP8/DP8-WideEP-MoRIIO-vllm-router" - agents: - queue: amd_mi350_ainic - timeout_in_minutes: 180 - concurrency: 2 - concurrency_group: "amd-disagg/mi350/2node" - commands: - - MODEL_NAME=GLM-5.2-FP8 NODES=2 GPUS_PER_NODE=8 WIDE_EP_MODE=1 MORIIO_READ_MODE=0 RUN_AFTER_HEALTH=accuracy ROUTER_TYPE=vllm-router WAIT=1 bash .buildkite/amd-disagg/run-slurm-disagg-test.sh diff --git a/.buildkite/amd-disagg/run-slurm-disagg-test.sh b/.buildkite/amd-disagg/run-slurm-disagg-test.sh deleted file mode 100644 index 6e062ec338..0000000000 --- a/.buildkite/amd-disagg/run-slurm-disagg-test.sh +++ /dev/null @@ -1,338 +0,0 @@ -#!/bin/bash -# ============================================================================= -# run-slurm-disagg-test.sh — foreground (login-node) submitter for the disagg -# P/D gate. The single command the Buildkite step runs. -# ----------------------------------------------------------------------------- -# Thin glue: submits run_xPyD_disagg.slurm. By default (WAIT=0, Spur-safe) it -# fire-and-forgets and returns 0 on a good submit; with WAIT=1 it polls the -# scheduler and exits with the job's exit code so a CI step pass/fails. -# stdout carries ONLY the SLURM job info -# (image/nodes line, job id, log path); after sbatch nothing else is written to -# stdout — the full job log persists at the --output path and pass/fail -# diagnostics go to stderr. The actual work lives in run_xPyD_disagg.slurm: it -# selects nodes, -# fans out one container per node via a single srun, and hands off to -# vllm_disagg.sh (rank-based prefill/decode self-select). -# -# Default target: 1P1D TP8 (NODES=2), nightly image, accuracy gate. -# -# Spur usage (fire-and-forget; the default here): -# bash run-slurm-disagg-test.sh # 1P1D TP8 -# WIDE_EP_MODE=1 bash run-slurm-disagg-test.sh # 1P1D wide-EP -# NODES=4 WIDE_EP_MODE=1 xP=2 yD=2 bash run-slurm-disagg-test.sh # 2P2D EP -# Prints the job id + NFS log paths and returns immediately (does NOT poll -# squeue/sacct, which hang on Spur). Track the run via the printed `tail -f`. -# -# Classic-Slurm / CI usage (block until done, exit with the gate's pass/fail): -# WAIT=1 bash run-slurm-disagg-test.sh -# ============================================================================= -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -JOB_SCRIPT="${JOB_SCRIPT:-${SCRIPT_DIR}/run_xPyD_disagg.slurm}" - -# ---- knobs (override from the Buildkite step env) -------------------------- -# Defaults tuned for the Spur AMD MI350X cluster -IMAGE="${IMAGE:-vllm/vllm-openai-rocm:nightly}" -NODES="${NODES:-2}" -GPUS_PER_NODE="${GPUS_PER_NODE:-8}" -PARTITION="${SLURM_PARTITION:-}" -TIME_LIMIT="${SLURM_TIME_LIMIT:-03:00:00}" -WIDE_EP_MODE="${WIDE_EP_MODE:-0}" # 0 -> 1P1D TP8 (default); 1 -> wide-EP -xP="${xP:-1}" -yD="${yD:-1}" -RUN_AFTER_HEALTH="${RUN_AFTER_HEALTH:-accuracy}" -HEALTH_TIMEOUT_S="${HEALTH_TIMEOUT_S:-3600}" # P/D bring-up budget; +900s grace must fit the CI step timeout -SHARED_MOUNT="${SHARED_MOUNT:-/data}" -LOG_ROOT="${LOG_ROOT:-${SHARED_MOUNT}/${USER:-$(whoami)}/disagg_logs}" -DRY_RUN="${DRY_RUN:-0}" -MORIIO_READ_MODE="${MORIIO_READ_MODE:-0}" - -WAIT="${WAIT:-0}" - -# ROUTER Type - defaults to vllm-router -ROUTER_TYPE="${ROUTER_TYPE:-vllm-router}" -ROUTER_PORT="${ROUTER_PORT:-30000}" -VLLM_ROUTER_IMAGE="${VLLM_ROUTER_IMAGE:-vllm/vllm-router:nightly}" -# Dry-run only validates wiring; cap its walltime low so it never holds the queue. -[[ "${DRY_RUN}" == "1" ]] && TIME_LIMIT="${SLURM_TIME_LIMIT:-00:10:00}" - -mkdir -p "${LOG_ROOT}" - -# Spur - sbatch scheduler. -DISAGG_SCRIPTS_STAGE="${DISAGG_SCRIPTS_STAGE:-/data/scratch/buildkite-agent}" -STAGED_DIR="${DISAGG_SCRIPTS_STAGE}/${BUILDKITE_COMMIT:-local}" -mkdir -p "${STAGED_DIR}" -cp -rL --no-preserve=ownership,timestamps "${SCRIPT_DIR}/." "${STAGED_DIR}/" -chmod -R u+rwX "${STAGED_DIR}" 2>/dev/null || true -export DISAGG_SCRIPTS_DIR="${STAGED_DIR}" -echo "[slurm-submit] staged scripts for compute nodes: ${DISAGG_SCRIPTS_DIR}" >&2 -export IMAGE MODEL_NAME WIDE_EP_MODE xP yD GPUS_PER_NODE RUN_AFTER_HEALTH HEALTH_TIMEOUT_S -export SHARED_MOUNT LOG_ROOT DRY_RUN MORIIO_READ_MODE -export ROUTER_TYPE ROUTER_PORT VLLM_ROUTER_IMAGE - -# Model selection. -[[ -n "${MODEL_NAME:-}" ]] && export MODEL_NAME -[[ -n "${MODEL_DIR:-}" ]] && export MODEL_DIR - -[[ -n "${PARTITION}" ]] && echo "[slurm-submit] NOTE: PARTITION='${PARTITION}' ignored (Spur sbatch has no --partition)" >&2 - -SUBMIT_SCRIPT="${JOB_SCRIPT}" -FILE_NODES="$(grep -oE '^#SBATCH[[:space:]]+--nodes=[0-9]+' "${JOB_SCRIPT}" | grep -oE '[0-9]+' | head -n1)" -if [[ -n "${NODES}" && -n "${FILE_NODES}" && "${NODES}" != "${FILE_NODES}" ]]; then - SUBMIT_SCRIPT="/tmp/$(basename "${JOB_SCRIPT%.slurm}")-n${NODES}-$$.slurm" - sed -E "s/^#SBATCH([[:space:]]+)--nodes=[0-9]+/#SBATCH\\1--nodes=${NODES}/" "${JOB_SCRIPT}" > "${SUBMIT_SCRIPT}" - echo "[slurm-submit] node count ${FILE_NODES}->${NODES}: submitting patched copy ${SUBMIT_SCRIPT}" >&2 -fi - -# Job name comes from the script's #SBATCH --job-name (SLURM sets SLURM_JOB_NAME -JOB_NAME="$(grep -oE '^#SBATCH[[:space:]]+--job-name=[^[:space:]]+' "${JOB_SCRIPT}" | sed -E 's/.*--job-name=//' | head -n1)" -JOB_NAME="${JOB_NAME:-vllm-disagg-pd}" - -echo "[slurm-submit] image=${IMAGE} nodes=${NODES} gpus/node=${GPUS_PER_NODE} mode=$([[ ${WIDE_EP_MODE} == 0 ]] && echo tp || echo ep) router=${ROUTER_TYPE} walltime=${TIME_LIMIT}" -SUBMIT_OUT="$(sbatch --time="${TIME_LIMIT}" "${SUBMIT_SCRIPT}")" -echo "${SUBMIT_OUT}" -# "Submitted batch job 114" -> 114 (last integer on the line). -JOB_ID="$(printf '%s\n' "${SUBMIT_OUT}" | grep -oE '[0-9]+' | tail -n1 || true)" -if [[ -z "${JOB_ID}" ]]; then - echo "[slurm-submit] ERROR: could not parse a job id from sbatch output above" >&2 - exit 1 -fi -echo "[slurm-submit] submitted job ${JOB_ID}" - -# The real per-job log is written to NFS from inside the job body by -# run_xPyD_disagg.slurm (`exec > "$LOG_ROOT/${SLURM_JOB_NAME}-${SLURM_JOB_ID}.log"`). -# SLURM_JOB_NAME resolves to our --job-name, so we can compute the path here. The -# per-role server/proxy/bench logs land under ${LOG_ROOT}/${JOB_ID}/. -LOG_FILE="${LOG_ROOT}/${JOB_NAME}-${JOB_ID}.log" -LOG_DIR="${LOG_ROOT}/${JOB_ID}" -echo "[slurm-submit] job log: ${LOG_FILE}" -echo "[slurm-submit] role logs: ${LOG_DIR}/" - -# --- Spur-safe by default: fire-and-forget -------------------------------------- -if [[ "${WAIT}" != "1" ]]; then - echo "[slurm-submit] submitted (WAIT=0, not polling). Track with:" >&2 - echo " tail -f ${LOG_FILE}" >&2 - echo " grep -aE '(PASS|FAIL): |exact_match' ${LOG_FILE}" >&2 - exit 0 -fi - -# --- WAIT=1: phase-aware poll of the NFS job log (scontrol-based) ------------- -# Detection is phased so failures surface fast instead of waiting out the full -# walltime: -# 1) submitted -> running : scontrol catches infra/scheduler failures -# (NODE_FAIL/BOOT_FAIL/CANCELLED/TIMEOUT/...) within -# one poll, or a stuck-PENDING/never-started job. -# 2) running -> healthy : advance once every endpoint reports healthy (+ the -# vllm-router, when used); fail on bring-up errors or -# the health budget. -# 3) healthy -> verdict : PASS/FAIL from the accuracy gate, capped. -# scontrol is the scheduler authority on this cluster -echo "[slurm-submit] WAIT=1: phase-aware poll of ${LOG_FILE} (timeout ${TIME_LIMIT})" >&2 - -_h=0 _m=0 _s=0 -IFS=: read -r _h _m _s <<< "${TIME_LIMIT}" -WAIT_DEADLINE=$(( $(date +%s) + 10#${_h}*3600 + 10#${_m}*60 + 10#${_s:-0} )) -unset -v _h _m _s - -# Per-phase budgets (all overridable from the Buildkite step env). -POLL_INTERVAL="${POLL_INTERVAL:-20}" -SUBMIT_GRACE_S="${SUBMIT_GRACE_S:-900}" # reach RUNNING within 15m -PENDING_MAX_S="${PENDING_MAX_S:-1800}" # tolerate 30m queued -HEALTH_PHASE_TIMEOUT_S="${HEALTH_PHASE_TIMEOUT_S:-$(( HEALTH_TIMEOUT_S + 900 ))}" -WORKLOAD_TIMEOUT_S="${WORKLOAD_TIMEOUT_S:-1800}" # accuracy/bench cap -COMPLETED_GRACE_S="${COMPLETED_GRACE_S:-60}" # verdict lag after job exit - -SENTINEL="${LOG_DIR}/.disagg_done" - -# scontrol field extractor (authoritative here; timeout-guarded so a momentary -# scheduler stall can't wedge the poll). Returns the value or "". -job_field() { # $1=jobid $2=field -> value | "" - timeout 15 scontrol show job "$1" 2>/dev/null \ - | grep -oE "$2=[^ ]+" | head -n1 | cut -d= -f2- || true -} -have() { grep -aqE "$1" "${LOG_FILE}" 2>/dev/null; } - -# The job's own verdict sources, in authority order: the .disagg_done sentinel -# (rank-0 rc, written by a single writer) and then the gate line in the log. -# Sets STATE/RC/REASON and returns 0 when a verdict exists, 1 when it does not. -read_verdict() { - if [[ -f "${SENTINEL}" ]]; then - RC="$(tr -dc '0-9' < "${SENTINEL}" 2>/dev/null || true)"; RC="${RC:-1}" - if [[ "${RC}" == "0" ]]; then STATE="COMPLETED"; else STATE="FAILED"; fi - REASON="sentinel"; return 0 - fi - if have '(PASS|FAIL): '; then - if have 'FAIL: '; then STATE="FAILED"; RC=1; else STATE="COMPLETED"; RC=0; fi - REASON="gate"; return 0 - fi - return 1 -} - -# Never let the job outlive this poller. Cancelling a Buildkite build kills the -# agent's bootstrap, and without this the sbatch job keeps its whole allocation -# until the walltime expires. Only armed on the WAIT=1 path — under WAIT=0, -# leaving the job running is the point. -CANCEL_GRACE_S="${CANCEL_GRACE_S:-120}" -_CLEANED=0 -# shellcheck disable=SC2329 # invoked from cleanup_job, which the traps below call -job_active() { - case "$(job_field "${JOB_ID}" JobState)" in - RUNNING|PENDING|COMPLETING|CONFIGURING|SUSPENDED|REQUEUED) return 0 ;; - *) return 1 ;; - esac -} -# shellcheck disable=SC2329 # invoked from the traps below -cleanup_job() { - [[ "${_CLEANED}" == "1" || -z "${JOB_ID:-}" ]] && return 0 - _CLEANED=1 - # A job that reported its own verdict is mid-teardown; give it a bounded - # moment to finish. Timeouts and infra failures get cancelled immediately — - # there is nothing to wait for and the nodes should come back now. - if [[ "${REASON:-}" == "sentinel" || "${REASON:-}" == "gate" || "${STATE:-}" == "COMPLETED" ]]; then - local deadline=$(( $(date +%s) + CANCEL_GRACE_S )) - while (( $(date +%s) < deadline )) && job_active; do sleep 5; done - fi - if job_active; then - echo "[slurm-submit] cleanup: cancelling job ${JOB_ID}" >&2 - scancel "${JOB_ID}" 2>/dev/null \ - || echo "[slurm-submit] WARN: scancel ${JOB_ID} failed; job may still hold nodes" >&2 - fi - return 0 -} -trap cleanup_job EXIT -trap 'cleanup_job; exit 130' INT -trap 'cleanup_job; exit 143' TERM -trap 'cleanup_job; exit 129' HUP - -STATE="" -RC=1 -REASON="" -PHASE="submitted" -T_PHASE=$(date +%s) - -while [[ $(date +%s) -lt ${WAIT_DEADLINE} ]]; do - NOW=$(date +%s) - - # (1) Ultimate authority: terminal sentinel (holds rank-0 rc), then the - # explicit accuracy gate line. Honored regardless of phase. - read_verdict && break - - # (2) Scheduler state via scontrol: drives phase transitions and catches - # infra/scheduler failures fast. - ST="$(job_field "${JOB_ID}" JobState)" - case "${ST}" in - RUNNING|COMPLETING) - if [[ "${PHASE}" == "submitted" ]]; then PHASE="bringup"; T_PHASE="${NOW}"; fi - ;; - NODE_FAIL|BOOT_FAIL|CANCELLED|TIMEOUT|OUT_OF_MEMORY|DEADLINE|PREEMPTED) - STATE="infra-${ST}"; RC=1 - REASON="scontrol Reason=$(job_field "${JOB_ID}" Reason)"; break - ;; - FAILED) - # Ambiguous (infra crash vs a legit accuracy exit=1). A gate/sentinel - # line would have won above, so classify by the phase we reached. - case "${PHASE}" in - submitted) STATE="infra-FAILED" ;; - bringup) STATE="server-failed" ;; - *) STATE="workload-failed" ;; - esac - RC=1; REASON="scontrol JobState=FAILED phase=${PHASE}"; break - ;; - COMPLETED) - _vd=$(( NOW + COMPLETED_GRACE_S )) - _found=0 - while :; do - if read_verdict; then _found=1; break; fi - (( $(date +%s) >= _vd )) && break - sleep 5 - done - if (( _found == 1 )); then - echo "[slurm-submit] verdict arrived after JobState=COMPLETED (${REASON})" >&2 - else - STATE="completed-no-verdict"; RC=1 - REASON="COMPLETED with no sentinel/gate within ${COMPLETED_GRACE_S}s (a walltime kill reports COMPLETED here)" - fi - break - ;; - PENDING|CONFIGURING|RESV_DEL_HOLD|REQUEUED) - if (( NOW - T_PHASE > PENDING_MAX_S )); then - STATE="infra-stuck-${ST}"; RC=1; REASON="queued > ${PENDING_MAX_S}s"; break - fi - ;; - "") - # scontrol lost the job (purged past MinJobAge) with no log verdict: - # rely on the terminal markers above + the per-phase deadlines below. - : - ;; - esac - - # (3) Log-driven phase progress + per-phase deadlines. Fast-fails hangs where - # the job is still RUNNING (so scontrol won't help) but bring-up/workload - # is stuck. - case "${PHASE}" in - submitted) - # Job body appearing in the log is a second "it started" signal for - # when scontrol is briefly empty. - if have 'Selected node IPs|health-gate:|\[disagg-pd\]'; then - PHASE="bringup"; T_PHASE="${NOW}" - elif [[ -z "${ST}" ]] && (( NOW - T_PHASE > SUBMIT_GRACE_S )); then - # scontrol can't confirm the job exists (empty state) AND no log - # output within the grace window -> treat as a lost/failed launch. - # NB: a genuinely-queued job reports PENDING via scontrol above and - # is governed by PENDING_MAX_S (default 30m), not this grace window. - STATE="infra-nostart"; RC=1; REASON="no scheduler state or log within ${SUBMIT_GRACE_S}s"; break - fi - ;; - bringup) - if have 'FAIL:|TIMEOUT waiting for|exited while waiting|Traceback \(most recent'; then - STATE="server-failed"; RC=1; REASON="bring-up failure in log"; break - fi - NEED="$(grep -aoE 'waiting on [0-9]+' "${LOG_FILE}" 2>/dev/null | grep -oE '[0-9]+' | tail -n1 || true)" - GOT="$(grep -acE '\] healthy: ' "${LOG_FILE}" 2>/dev/null || true)"; GOT="${GOT:-0}" - ROUTER_OK=1 - if [[ "${ROUTER_TYPE}" == "vllm-router" ]]; then - if ! have 'vllm-router healthy on'; then ROUTER_OK=0; fi - fi - if [[ -n "${NEED}" ]] && (( GOT >= NEED )) && (( ROUTER_OK == 1 )); then - PHASE="workload"; T_PHASE="${NOW}" - elif (( NOW - T_PHASE > HEALTH_PHASE_TIMEOUT_S )); then - STATE="bringup-timeout"; RC=1; REASON="no healthy within ${HEALTH_PHASE_TIMEOUT_S}s"; break - fi - ;; - workload) - # PASS/FAIL handled at the top; only enforce the cap here. - if (( NOW - T_PHASE > WORKLOAD_TIMEOUT_S )); then - STATE="workload-timeout"; RC=1; REASON="no verdict within ${WORKLOAD_TIMEOUT_S}s"; break - fi - ;; - esac - - sleep "${POLL_INTERVAL}" -done - -if [[ -z "${STATE}" ]]; then - echo "[slurm-submit] WARN: no verdict before ${TIME_LIMIT}; failing" >&2 - STATE="deadline"; RC=1; REASON="poll deadline" -fi - -# Pre-flight logic. -PF_LINES=() -if compgen -G "${LOG_DIR}/preflight_NODE*.log" >/dev/null 2>&1; then - cat -- "${LOG_DIR}"/preflight_NODE*.log >&2 || true - mapfile -t PF_LINES < <(grep -h '^PREFLIGHT-REJECTED: ' "${LOG_DIR}"/preflight_NODE*.log 2>/dev/null || true) -fi -if (( ${#PF_LINES[@]} > 0 )) && (( RC != 0 )); then - STATE="preflight-rejected" - REASON="${PF_LINES[0]#PREFLIGHT-REJECTED: }" - if (( ${#PF_LINES[@]} > 1 )); then - REASON="${REASON} [+$(( ${#PF_LINES[@]} - 1 )) more node(s), see ${LOG_DIR}/]" - fi -fi - -# Surface the accuracy gate verdict (if any) from the job log — to stderr. -GATE_LINE=$(grep -aE '(PASS|FAIL): ' "${LOG_FILE}" 2>/dev/null | tail -n1 || true) -[[ -n "${GATE_LINE}" ]] && echo "[slurm-submit] gate: ${GATE_LINE}" >&2 - -echo "[slurm-submit] job ${JOB_ID} finished: state=${STATE:-unknown} phase=${PHASE} exit=${RC} reason=${REASON:-}" >&2 -exit "${RC}" diff --git a/.buildkite/amd-disagg/run_xPyD_disagg.slurm b/.buildkite/amd-disagg/run_xPyD_disagg.slurm deleted file mode 100644 index c5513b699e..0000000000 --- a/.buildkite/amd-disagg/run_xPyD_disagg.slurm +++ /dev/null @@ -1,535 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=vllm-disagg-pd -#SBATCH --nodes=2 -#SBATCH --ntasks-per-node=1 -#SBATCH --gres=gpu:8 -#SBATCH --exclusive -#SBATCH --chdir=/tmp -#SBATCH --output=/tmp/spur-%j.out -#SBATCH --error=/tmp/spur-%j.err - -set -euo pipefail - - -# ---- inputs (env > default; forwarded by run-slurm-disagg-test.sh via --export) -IMAGE="${IMAGE:-vllm/vllm-openai-rocm:nightly}" -WIDE_EP_MODE="${WIDE_EP_MODE:-0}" -xP="${xP:-1}" -yD="${yD:-1}" -GPUS_PER_NODE="${GPUS_PER_NODE:-8}" - -SHARED_MOUNT="${SHARED_MOUNT:-/data}" -LOG_ROOT="${LOG_ROOT:-/data/${USER:-$(id -un)}/disagg_logs}" -RUN_AFTER_HEALTH="${RUN_AFTER_HEALTH:-accuracy}" # bench | accuracy -DRY_RUN="${DRY_RUN:-0}" -MORIIO_READ_MODE="${MORIIO_READ_MODE:-0}" # 0 -> prefill writes (default); 1 -> decode reads KV (read_mode) - -# ---- router / gateway (host-side; also forwarded into the container) -------- -# vllm-router -> external `vllm/vllm-router` container started on the rank-0 node (default) -# proxy -> in-container MoRIIO proxy (no extra container) -ROUTER_TYPE="${ROUTER_TYPE:-vllm-router}" -ROUTER_PORT="${ROUTER_PORT:-30000}" -ROUTER_POLICY="${ROUTER_POLICY:-round_robin}" -VLLM_ROUTER_IMAGE="${VLLM_ROUTER_IMAGE:-vllm/vllm-router:nightly}" -PROXY_PING_PORT="${PROXY_PING_PORT:-36367}" # must equal vllm-router --vllm-discovery-address port -PROXY_PORT="${PROXY_PORT:-10001}" -# Intra-node data-parallel size the router uses to route the MoRIIO KV-notify to -# the right DP rank. Required for wideEP DP (else decode hangs: "remote blocks -# never arrived"). EP -> GPUS_PER_NODE (matches --data-parallel-size-local); TP -> 1. -if [[ "${WIDE_EP_MODE}" == "1" ]]; then - ROUTER_DP_LOCAL="${ROUTER_DP_LOCAL:-${GPUS_PER_NODE}}" -else - ROUTER_DP_LOCAL="${ROUTER_DP_LOCAL:-1}" -fi -if [[ "${ROUTER_TYPE}" == "vllm-router" ]]; then - GATEWAY_PORT="${GATEWAY_PORT:-${ROUTER_PORT}}" -else - GATEWAY_PORT="${GATEWAY_PORT:-${PROXY_PORT}}" -fi - -# This line is a recommended template from infra side. -exec > "$LOG_ROOT/${SLURM_JOB_NAME}-${SLURM_JOB_ID}.log" 2>&1 - -# Where the disagg scripts live ON THE HOST (bind-mounted into the container). -# Spur runs the job body from /tmp, so $(pwd)/SLURM_SUBMIT_DIR aren't reliable; -# default to the known host scripts dir. Override via env if you relocate it. -DISAGG_SCRIPTS_DIR="${DISAGG_SCRIPTS_DIR:-${SLURM_SUBMIT_DIR:-$(pwd)}}" -echo "DISAGG SCRIPTS DIR is ----" -echo "$DISAGG_SCRIPTS_DIR" - -# In-container path the scripts are mounted to (and sourced from). -CONTAINER_SCRIPTS="${CONTAINER_SCRIPTS:-/vllm-workspace/.buildkite/amd-disagg}" -CLUSTER_ENV_IN="${CLUSTER_ENV_IN:-${CONTAINER_SCRIPTS}/cluster.sh}" - - -# Health endpoints (TP-mode ports from cluster.sh defaults). -PREFILL_PORT="${PREFILL_PORT:-8100}" -DECODE_PORT="${DECODE_PORT:-8200}" -HEALTH_TIMEOUT_S="${HEALTH_TIMEOUT_S:-3600}" - -# EP-mode ports, mirrored from cluster.sh purely so the preflight below can test -# them host-side; the in-container values still come from cluster.sh. -SERVE_PORT="${SERVE_PORT:-20005}" -RPC_PORT="${RPC_PORT:-13345}" -KV_PORT="${KV_PORT:-9711}" -LOCAL_PING_PORT="${LOCAL_PING_PORT:-61555}" -# vllm-router binds this for Prometheus in addition to ROUTER_PORT, and panics -# at startup if it is taken. -ROUTER_METRICS_PORT="${ROUTER_METRICS_PORT:-29000}" - -# Node preflight (see preflight_node): 0 disables, VRAM% above which a GPU counts -# as busy, and how long to let a reaped container's VRAM drain before judging. -PREFLIGHT="${PREFLIGHT:-1}" -PREFLIGHT_MAX_VRAM_PCT="${PREFLIGHT_MAX_VRAM_PCT:-10}" -PREFLIGHT_GPU_SETTLE_S="${PREFLIGHT_GPU_SETTLE_S:-30}" - - -NUM_NODES=$((xP + yD)) -echo "Calculated NUM_NODES: $NUM_NODES (xP=$xP + yD=$yD) WIDE_EP_MODE=$WIDE_EP_MODE" - -# ------------------------ -# Extract NUM_NODES from SLURM allocation and select nodes -# ------------------------ -echo "Original SLURM allocation:" -echo "SLURM_JOB_NODELIST: $SLURM_JOB_NODELIST" -echo "SLURM_NNODES: $SLURM_NNODES" -echo "SLURM_NTASKS: $SLURM_NTASKS" - -# scontrol is available on classic Slurm but not in the Spur job body. Use it to -# expand compressed ranges when present, else split the (already comma-separated, -# full-hostname) SLURM_JOB_NODELIST on commas. -if command -v scontrol >/dev/null 2>&1; then - FULL_NODELIST=$(scontrol show hostnames "$SLURM_JOB_NODELIST") -else - FULL_NODELIST=$(echo "$SLURM_JOB_NODELIST" | tr ',' '\n') -fi -AVAILABLE_COUNT=$(echo "$FULL_NODELIST" | wc -l) - -if (( AVAILABLE_COUNT < NUM_NODES )); then - echo "ERROR: Need ${NUM_NODES} nodes (xP=${xP} + yD=${yD}) but only ${AVAILABLE_COUNT} allocated." >&2 - exit 1 -fi - -# Pick the first NUM_NODES nodes and order them to match how IPs get assigned: -# * classic Slurm: sort alphabetically -- srun assigns PROCID=0 to the -# alphabetically-first hostname, so IPS[] (built per node below) lines up. -# * Spur: preserve RANK order (SPUR_NODELIST), so the hostnames pair 1:1 with -# the rank-ordered IPs in SPUR_PEER_NODES; an alphabetical sort here would -# desync MASTER_NODE / MASTER_ADDR / IPADDRS from the actual ranks. -# IPADDRS[0..xP-1] = prefill, IPADDRS[xP..] = decode, so this order must match. -if command -v srun >/dev/null 2>&1; then - SELECTED_NODES=$(echo "$FULL_NODELIST" | head -n "$NUM_NODES" | sort) -elif [[ -n "${SPUR_NODELIST:-}" ]]; then - SELECTED_NODES=$(echo "${SPUR_NODELIST}" | tr ',' '\n' | sed '/^$/d' | head -n "$NUM_NODES") -else - SELECTED_NODES=$(echo "$FULL_NODELIST" | head -n "$NUM_NODES") -fi -SELECTED_NODELIST_STR=$(echo "$SELECTED_NODES" | tr '\n' ',' | sed 's/,$//') - -# Create new nodelist in SLURM format -# This is a simplified approach - for complex ranges, you might need more sophisticated parsing -NEW_SLURM_NODELIST=$(echo "$SELECTED_NODES" | paste -sd, | sed 's/,/,/g') - -# Update SLURM environment variables -export SLURM_NNODES=$NUM_NODES -export SLURM_NTASKS=$NUM_NODES -export SLURM_JOB_NUM_NODES=$NUM_NODES -export SLURM_NPROCS=$NUM_NODES -export SLURM_JOB_NODELIST="$NEW_SLURM_NODELIST" -export SLURM_NODELIST="$NEW_SLURM_NODELIST" - -# Keep other SLURM variables as they were or set defaults -export SLURM_TASKS_PER_NODE="1(x$NUM_NODES)" -export SLURM_SUBMIT_DIR="${SLURM_SUBMIT_DIR:-${HOME}}" -export SLURM_CLUSTER_NAME="${SLURM_CLUSTER_NAME:-vllm-di-ci}" -export SLURM_JOB_CPUS_PER_NODE="${SLURM_JOB_CPUS_PER_NODE:-1}" -export SLURM_JOB_PARTITION="${SLURM_JOB_PARTITION:-amd-rccl}" -export SLURM_JOBID="${SLURM_JOBID:-$SLURM_JOB_ID}" -export SLURM_JOB_QOS="${SLURM_JOB_QOS:-normal}" -export SLURM_JOB_ACCOUNT="${SLURM_JOB_ACCOUNT:-amd-rccl}" -export SLURM_NTASKS_PER_NODE=1 -export SLURM_SUBMIT_HOST="${SLURM_SUBMIT_HOST:-$(hostname)}" -export SLURM_JOB_ID="${SLURM_JOB_ID:-}" -export SLURM_CONF="${SLURM_CONF:-/etc/slurm/slurm.conf}" -export SLURM_JOB_NAME="${SLURM_JOB_NAME:-${xP}p${yD}d_bench-serving}" - -echo "" -echo "Updated SLURM Environment Variables:" -echo "SLURM_JOB_ID: $SLURM_JOB_ID" -echo "SLURM_JOB_NODELIST: $SLURM_JOB_NODELIST" -echo "SLURM_NNODES: $SLURM_NNODES" -echo "SLURM_NTASKS: $SLURM_NTASKS" -echo "SLURM_TASKS_PER_NODE: $SLURM_TASKS_PER_NODE" -echo "SLURM_JOB_CPUS_PER_NODE: $SLURM_JOB_CPUS_PER_NODE" -echo "SLURM_JOB_PARTITION: $SLURM_JOB_PARTITION" -echo "SLURM_JOB_NUM_NODES: $SLURM_JOB_NUM_NODES" -echo "SLURM_JOBID: $SLURM_JOBID" -echo "SLURM_JOB_QOS: $SLURM_JOB_QOS" -echo "SLURM_NODELIST: $SLURM_NODELIST" -echo "SLURM_JOB_ACCOUNT: $SLURM_JOB_ACCOUNT" -echo "SLURM_NPROCS: $SLURM_NPROCS" -echo "SLURM_SUBMIT_HOST: $SLURM_SUBMIT_HOST" -echo "SLURM_CONF: $SLURM_CONF" -echo "SLURM_JOB_NAME: $SLURM_JOB_NAME" -echo "SLURM_NTASKS_PER_NODE: $SLURM_NTASKS_PER_NODE" -echo "SLURM_SUBMIT_DIR: $SLURM_SUBMIT_DIR" -echo "SLURM_CLUSTER_NAME: $SLURM_CLUSTER_NAME" -echo "ulimit: $(ulimit -a)" -echo "" -echo "Selected nodes for execution:" -echo "$SELECTED_NODES" -echo "" - -# Node information -MASTER_NODE=$(echo "$SELECTED_NODES" | head -n 1) -MASTER_PORT=39566 # Choose an open port - -IPS=() -if command -v srun >/dev/null 2>&1; then - # Classic Slurm: resolve each node's IP via srun (original behavior). - for NODE in $SELECTED_NODES; do - IP=$(srun --nodes=1 --ntasks=1 --time=00:20:00 --nodelist="$NODE" bash -c 'hostname -I' | awk 'NR==1 {print $1}') - IPS+=("$IP") - done -else - # Spur job body (no srun): use the rank-ordered IPs Spur provides in - # SPUR_PEER_NODES ("ip:port,ip:port,..."). These are the cross-node-routable - # bootstrap addresses; hostnames resolve to public IPs and the local node to - # 127.0.1.1, so DNS is not usable here. - mapfile -t IPS < <(echo "${SPUR_PEER_NODES:-}" | tr ',' '\n' | cut -d: -f1 | sed '/^$/d') - if (( ${#IPS[@]} == 0 )); then - echo "ERROR: no srun and SPUR_PEER_NODES is empty; cannot determine node IPs." >&2 - exit 1 - fi -fi -MASTER_ADDR="${IPS[0]}" - -IPADDRS="${IPS[*]}" -IPADDRS="${IPADDRS// /,}" -echo "Selected node IPs: ${IPADDRS}" - -NNODES=$NUM_NODES - -echo "MASTER_NODE is ${MASTER_NODE}" -echo "MASTER_ADDR is ${MASTER_ADDR}" -echo "MASTER_PORT is ${MASTER_PORT}" -echo "NNODES is ${NNODES}" - -# Per-job log folder (MAD-style): all per-role server/proxy/bench logs -export LOG_PATH="${LOG_ROOT}/${SLURM_JOB_ID}" -mkdir -p "$LOG_PATH" -echo "[disagg-pd] LOG_PATH=$LOG_PATH" - -# ---------------------------------------------------------------------------- -# Launch one container per node via a single srun and hand off to -# vllm_disagg.sh node (rank-based self-select). -# ---------------------------------------------------------------------------- -# IPADDRS is the ordered node IP list (prefill IPs first, then decode). IPS[] -# was built in SELECTED_NODES (alphabetical) order — the same order srun assigns -# SLURM_PROCID — so the rank->role mapping lines up: -# rank 0 -> prefill master + MoRIIO proxy + orchestrator -# rank 1 .. xP-1 -> prefill -# rank xP .. end -> decode -export IPADDRS - -export WIDE_EP_MODE xP yD GPUS_PER_NODE RUN_AFTER_HEALTH HEALTH_TIMEOUT_S DRY_RUN MORIIO_READ_MODE -export IMAGE SHARED_MOUNT CONTAINER_SCRIPTS DISAGG_SCRIPTS_DIR -export CLUSTER_ENV="${CLUSTER_ENV_IN}" -export DOCKER_CONT_NAME="vllm_disagg_${MODEL_NAME:-model}_${SLURM_JOB_ID}" - -# Router env: forwarded into the main container (cluster.sh honors these) and used -# host-side to launch the external router container on rank 0. -export ROUTER_TYPE ROUTER_PORT ROUTER_POLICY VLLM_ROUTER_IMAGE PROXY_PING_PORT GATEWAY_PORT ROUTER_DP_LOCAL -export ROUTER_CONT_NAME="vllm_router_${SLURM_JOB_ID}" - -# Consumed by preflight_node on each node (srun forwards the environment). -export PREFILL_PORT DECODE_PORT PROXY_PORT ROUTER_METRICS_PORT -export SERVE_PORT RPC_PORT KV_PORT LOCAL_PING_PORT -export PREFLIGHT PREFLIGHT_MAX_VRAM_PCT PREFLIGHT_GPU_SETTLE_S - -echo "[disagg-pd] IPADDRS=${IPADDRS}" -echo "[disagg-pd] launching ${NUM_NODES} containers (1/node) -> vllm_disagg.sh node" - -# Refuse a node that is not actually free. $1 = this node's global rank. -# -# `#SBATCH --exclusive` only keeps out other SLURM jobs. A plain `docker run` on -# a node is invisible to the scheduler, so we get allocated nodes whose GPUs and -# ports are already in use. Without this the job pulls the image, loads the model -# and dies minutes later inside vLLM ("Free memory on device cuda:N ... is less -# than desired GPU memory utilization"); worse, on rank 0 a foreign router -# already listening on ROUTER_PORT can satisfy the health gate and take the -# accuracy traffic. Called after the stale-container reap, with a settle window -# so VRAM the reap is still releasing is not counted against us. -preflight_node() { - local rank="$1" - if [ "${PREFLIGHT:-1}" != "1" ]; then - echo "Rank $rank preflight: skipped (PREFLIGHT=0)" - return 0 - fi - - local host max_pct settle problems="" - local -a issues=() # one short tag per fault, for the one-line summary - host="$(hostname)" - max_pct="${PREFLIGHT_MAX_VRAM_PCT:-10}" - settle="${PREFLIGHT_GPU_SETTLE_S:-30}" - - # GPUs: vLLM asks for a fraction of TOTAL, so memory already resident is - # subtracted from what it can get and pushes it under its own threshold. - if command -v rocm-smi >/dev/null 2>&1; then - local csv busy nbusy ncards deadline - deadline=$(( $(date +%s) + settle )) - while :; do - csv="$(rocm-smi --showmeminfo vram --csv 2>/dev/null || true)" - busy="$(printf '%s\n' "$csv" | - awk -F, -v m="$max_pct" '/^card/ && $2+0 > 0 { - pct = $3 * 100 / $2 - if (pct > m) printf "%s=%d%%(%.0fGiB) ", $1, pct, $3/1073741824 - }' || true)" - if [ -z "$busy" ]; then break; fi - if [ "$(date +%s)" -ge "$deadline" ]; then break; fi - sleep 5 - done - if [ -n "$busy" ]; then - nbusy="$(printf '%s' "$busy" | tr ' ' '\n' | grep -c 'card' || true)" - ncards="$(printf '%s\n' "$csv" | grep -c '^card' || true)" - issues+=("${nbusy}/${ncards} GPUs hold VRAM above ${max_pct}%") - problems="${problems} GPU VRAM in use above ${max_pct}%: ${busy}"$'\n' - fi - else - echo "Rank $rank preflight: no rocm-smi on $host, skipping GPU check" - fi - - # Ports this rank will bind. Every container here is --network host, so a - # foreign listener either collides or silently serves our traffic. - local -a want=() - if [ "${WIDE_EP_MODE:-0}" = "1" ]; then - want+=("$SERVE_PORT" "$RPC_PORT" "$KV_PORT" "$LOCAL_PING_PORT") - elif [ "$rank" -lt "${xP:-1}" ]; then - want+=("$PREFILL_PORT") - else - want+=("$DECODE_PORT") - fi - if [ "$rank" = "0" ]; then - want+=("$PROXY_PING_PORT") - if [ "$ROUTER_TYPE" = "vllm-router" ]; then - want+=("$ROUTER_PORT" "$ROUTER_METRICS_PORT") - else - want+=("$PROXY_PORT") - fi - fi - - if command -v ss >/dev/null 2>&1; then - local bound taken="" p - bound="$(ss -lnt 2>/dev/null | awk '{ n = split($4, a, ":"); print a[n] }' | sort -u || true)" - for p in "${want[@]}"; do - if printf '%s\n' "$bound" | grep -qx -- "$p"; then - taken="${taken}${p} " - fi - done - if [ -n "$taken" ]; then - issues+=("ports already bound: ${taken% }") - problems="${problems} ports already bound: ${taken}"$'\n' - fi - else - echo "Rank $rank preflight: no ss on $host, skipping port check" - fi - - if [ -n "$problems" ]; then - mkdir -p "$LOG_PATH" 2>/dev/null || true - local summary - summary="$(printf '%s; ' "${issues[@]}")" - summary="${summary%; }" - # Every node writes the shared job log concurrently, so this diagnostic can - # be overwritten there. Keep a single-writer copy per node as well, or the - # job fails in seconds with no surviving explanation. The submitter reads - # the PREFLIGHT-REJECTED line back out of it to use as the failure reason, - # so keep that prefix in sync with run-slurm-disagg-test.sh. It must not - # contain "FAIL:" — that is the accuracy gate's pattern. - { - echo "PREFLIGHT-REJECTED: rank ${rank} on ${host}: ${summary}" - printf '%s' "$problems" - echo " containers here: $(docker ps --format '{{.Names}} ({{.Image}})' 2>/dev/null | paste -sd'; ' - || true)" - echo " --exclusive does not exclude non-SLURM workloads; clear the node or exclude it from the allocation." - echo " Set PREFLIGHT=0 to submit anyway." - } | tee -a "$LOG_PATH/preflight_NODE${rank}.log" 2>/dev/null || true - # Unblock the other ranks: they park on this sentinel, and the submitter - # polls it. Without it a peer waits out HEALTH_TIMEOUT_S for an endpoint - # that is never coming. - echo 1 > "$LOG_PATH/.disagg_done" 2>/dev/null || true - return 1 - fi - - echo "Rank $rank preflight ok on $host: GPUs idle, ports free (${want[*]})" - return 0 -} - -# Per-node container launch, auto-dispatched by srun availability: -# Launch this node's disagg container and hand off to vllm_disagg.sh. $1 = this -# node's global rank. Relies on the env exported above (forwarded by srun on -# classic Slurm; already in scope when called directly on Spur). -run_node_container() { - local rank="$1" - set -o pipefail # propagate docker run exit code through the tee pipe - echo "Rank $rank on $(hostname)" - local -a stale=() - mapfile -t stale < <(docker ps -aq --filter name=vllm_disagg_ --filter name=vllm_router_ 2>/dev/null || true) - if (( ${#stale[@]} > 0 )); then - echo "Reaping stale containers on $(hostname): ${stale[*]}" - docker stop -t 30 "${stale[@]}" >/dev/null 2>&1 || true - docker rm "${stale[@]}" >/dev/null 2>&1 || true - fi - - preflight_node "$rank" || return 1 - - # Bind-mount the host AMD Pensando (ionic) RoCE userspace provider into the - # container IF present: the image ships an ABI-incompatible copy and the NICs - # are invisible without the host libs. Each entry is added only when the host - # file exists, so this is a no-op on non-ionic (e.g. mlx5) clusters. - local IONIC_MOUNTS=() - local host_ionic - host_ionic=$(readlink -f /usr/lib/x86_64-linux-gnu/libionic.so.1 2>/dev/null || true) - [ -n "$host_ionic" ] && IONIC_MOUNTS+=(-v "$host_ionic:/usr/lib/x86_64-linux-gnu/libionic.so.1:ro") - [ -f /usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav34.so ] && \ - IONIC_MOUNTS+=(-v "/usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav34.so:/usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav34.so:ro") - [ -f /etc/libibverbs.d/ionic.driver ] && \ - IONIC_MOUNTS+=(-v "/etc/libibverbs.d/ionic.driver:/etc/libibverbs.d/ionic.driver:ro") - echo "Rank $rank ionic mounts: ${IONIC_MOUNTS[*]:-}" - - # Pull latest container. - if [ "$DRY_RUN" != "1" ]; then - echo "Rank $rank pulling image on $(hostname): $IMAGE" - docker pull "$IMAGE" || echo "WARN: docker pull failed on $(hostname) for $IMAGE (using local copy if present)" - if [ "$ROUTER_TYPE" = "vllm-router" ] && [ "$rank" = "0" ]; then - echo "Rank $rank pulling router image on $(hostname): $VLLM_ROUTER_IMAGE" - docker pull "$VLLM_ROUTER_IMAGE" || echo "WARN: docker pull failed on $(hostname) for $VLLM_ROUTER_IMAGE (using local copy if present)" - fi - fi - - # Start the external vLLM router as a SEPARATE container on the rank-0 node. - if [ "$ROUTER_TYPE" = "vllm-router" ] && [ "$rank" = "0" ] && [ "$DRY_RUN" = "1" ]; then - echo "DRY_RUN: would start vllm-router on $(hostname): $VLLM_ROUTER_IMAGE (discovery :$PROXY_PING_PORT, port :$ROUTER_PORT, policy $ROUTER_POLICY, dp_local=$ROUTER_DP_LOCAL)" - elif [ "$ROUTER_TYPE" = "vllm-router" ] && [ "$rank" = "0" ]; then - mkdir -p "$LOG_PATH" 2>/dev/null || true - { - echo "Starting vllm-router on $(hostname): $VLLM_ROUTER_IMAGE (discovery :$PROXY_PING_PORT, port :$ROUTER_PORT, policy $ROUTER_POLICY, dp_local=$ROUTER_DP_LOCAL)" - docker run -d \ - --name "$ROUTER_CONT_NAME" \ - --network host \ - --user "$(id -u):$(id -g)" \ - -v "$LOG_PATH":/run_logs \ - -v "$SHARED_MOUNT":"$SHARED_MOUNT" \ - -v "/data/$(id -un):/workspace" \ - -e HOME=/workspace \ - -e USER="$(id -un)" \ - -e LOGNAME="$(id -un)" \ - "$VLLM_ROUTER_IMAGE" \ - bash -lc "exec vllm-router --vllm-pd-disaggregation --kv-connector moriio --vllm-discovery-address 0.0.0.0:$PROXY_PING_PORT --port $ROUTER_PORT --host 0.0.0.0 --intra-node-data-parallel-size $ROUTER_DP_LOCAL --policy $ROUTER_POLICY --prefill-policy $ROUTER_POLICY --decode-policy $ROUTER_POLICY --log-level info >> $LOG_PATH/vllm_router.log 2>&1" \ - && echo "vllm-router container started ok" \ - || echo "WARN: failed to start vllm-router container on $(hostname)" - } 2>&1 | tee -a "$LOG_PATH/vllm_router_start.log" - fi - - docker run --rm \ - --user "$(id -u):$(id -g)" \ - --device /dev/dri \ - --device /dev/kfd \ - --device /dev/infiniband \ - --network host \ - --ipc host \ - --group-add "$(getent group video | cut -d: -f3)" \ - --group-add "$(getent group render | cut -d: -f3)" \ - --cap-add SYS_PTRACE \ - --cap-add IPC_LOCK \ - --security-opt seccomp=unconfined \ - --shm-size 64G \ - --ulimit nofile=1048576:1048576 \ - -v "$SHARED_MOUNT":"$SHARED_MOUNT" \ - -v "$DISAGG_SCRIPTS_DIR":"$CONTAINER_SCRIPTS" \ - -v "/data/$(id -un):/workspace" \ - "${IONIC_MOUNTS[@]}" \ - --ulimit memlock=-1:-1 \ - -e HF_TOKEN \ - -e HOME=/workspace \ - -e USER="$(id -un)" \ - -e LOGNAME="$(id -un)" \ - -e TRITON_CACHE_DIR=/tmp/triton_cache \ - -e TORCHINDUCTOR_CACHE_DIR=/tmp/inductor_cache \ - -e VLLM_CACHE_ROOT=/tmp/vllm_cache \ - -e AITER_JIT_DIR=/tmp/aiter_jit \ - -e FLYDSL_RUNTIME_CACHE_DIR=/tmp/flydsl_cache \ - -e SLURM_JOB_ID="$SLURM_JOB_ID" \ - -e SLURM_JOB_NODELIST="$SLURM_JOB_NODELIST" \ - -e NODE_RANK="$rank" \ - -e IPADDRS="$IPADDRS" \ - -e WIDE_EP_MODE="$WIDE_EP_MODE" \ - -e xP="$xP" -e yD="$yD" \ - -e GPUS_PER_NODE="$GPUS_PER_NODE" \ - -e CLUSTER_ENV="$CLUSTER_ENV" \ - -e MODEL_NAME="${MODEL_NAME:-}" \ - -e MODEL_DIR="${MODEL_DIR:-}" \ - -e LOG_PATH="$LOG_PATH" \ - -e RUN_AFTER_HEALTH="$RUN_AFTER_HEALTH" \ - -e HEALTH_TIMEOUT_S="$HEALTH_TIMEOUT_S" \ - -e DRY_RUN="$DRY_RUN" \ - -e MORIIO_READ_MODE="$MORIIO_READ_MODE" \ - -e ROUTER_TYPE="$ROUTER_TYPE" \ - -e ROUTER_PORT="$ROUTER_PORT" \ - -e ROUTER_POLICY="$ROUTER_POLICY" \ - -e GATEWAY_PORT="$GATEWAY_PORT" \ - --name "$DOCKER_CONT_NAME" \ - --entrypoint /bin/bash \ - "$IMAGE" -c "bash $CONTAINER_SCRIPTS/vllm_disagg.sh node" \ - 2>&1 | tee "$LOG_PATH/main_NODE${rank}.log" - local node_rc=${PIPESTATUS[0]} # docker run exit code (not tee), for rank-0 pass/fail - # Stop the external router (rank-0 node) once the main container returns. - if [ "$ROUTER_TYPE" = "vllm-router" ] && [ "$rank" = "0" ]; then - docker stop "$ROUTER_CONT_NAME" 2>/dev/null || true; docker rm "$ROUTER_CONT_NAME" 2>/dev/null || true; - fi - return "$node_rc" -} - -# Teardown of the main + external router containers on every node ($1 = docker -# stop timeout). `docker run` containers outlive the client that started them, -# so on scancel the srun tasks die but the containers keep the GPUs until the -# walltime expires unless they are stopped explicitly. -teardown_containers() { - export TEARDOWN_STOP_T="${1:-30}" - if command -v srun >/dev/null 2>&1; then - # shellcheck disable=SC2016 # must expand on the remote node, not here - srun --nodelist="$SELECTED_NODELIST_STR" \ - --nodes="$NUM_NODES" --ntasks="$NUM_NODES" --ntasks-per-node=1 \ - bash -c 'docker stop -t "${TEARDOWN_STOP_T:-30}" "$DOCKER_CONT_NAME" >/dev/null 2>&1 || true; docker rm "$DOCKER_CONT_NAME" >/dev/null 2>&1 || true - [ "$ROUTER_TYPE" = "vllm-router" ] && { docker stop -t "${TEARDOWN_STOP_T:-30}" "$ROUTER_CONT_NAME" >/dev/null 2>&1 || true; docker rm "$ROUTER_CONT_NAME" >/dev/null 2>&1 || true; }' || true - else - docker stop -t "$TEARDOWN_STOP_T" "$DOCKER_CONT_NAME" >/dev/null 2>&1 || true - docker rm "$DOCKER_CONT_NAME" >/dev/null 2>&1 || true - if [ "$ROUTER_TYPE" = "vllm-router" ]; then - docker stop -t "$TEARDOWN_STOP_T" "$ROUTER_CONT_NAME" >/dev/null 2>&1 || true - docker rm "$ROUTER_CONT_NAME" >/dev/null 2>&1 || true - fi - fi - return 0 -} - -# SLURM sends SIGTERM then SIGKILL after KillWait (30s by default), so the trap -# uses a short stop timeout to fit inside that window. -trap 'echo "[disagg-pd] signal received, tearing down containers"; teardown_containers 10; exit 143' TERM INT - -RC=0 -if command -v srun >/dev/null 2>&1; then - # classic Slurm: fan out one task/node; ship the fn and call it with the - # srun-supplied per-task SLURM_PROCID (escaped so the remote shell expands it). - srun --label \ - --nodelist="$SELECTED_NODELIST_STR" \ - --nodes="$NUM_NODES" --ntasks="$NUM_NODES" --ntasks-per-node=1 \ - bash -c "$(declare -f preflight_node run_node_container); run_node_container \"\$SLURM_PROCID\"" || RC=$? -else - # Spur: the body already runs once per node; this node's rank is - # SPUR_TASK_OFFSET (fall back to NODE_RANK, then 0 for a single-node run). - run_node_container "${SPUR_TASK_OFFSET:-${NODE_RANK:-0}}" || RC=$? -fi - -teardown_containers 30 - -echo "[disagg-pd] launch finished rc=${RC}; logs in ${LOG_PATH}" -exit "${RC}" diff --git a/.buildkite/amd-disagg/vllm_disagg.sh b/.buildkite/amd-disagg/vllm_disagg.sh deleted file mode 100644 index 0974862a38..0000000000 --- a/.buildkite/amd-disagg/vllm_disagg.sh +++ /dev/null @@ -1,635 +0,0 @@ -#!/bin/bash -# ============================================================================= -# vLLM disaggregated (P/D) launcher (rank-based / SLURM-native). -# ----------------------------------------------------------------------------- -# -# rank 0 .. xP-1 -> prefill (rank 0 is also the orchestrator + proxy) -# rank xP .. xP+yD-1 -> decode -# -# Rank 0 additionally: starts the MoRIIO proxy, health-gates every server, -# runs the post-health workload (RUN_AFTER_HEALTH=bench|accuracy|none), then -# drops a shared-FS completion sentinel so the other ranks shut down and the -# single `srun` returns. Rank 0's exit code is the run's pass/fail. -# -# Explicit per-role invocation is also supported (granular for manual use): -# proxy | prefill | decode | bench | accuracy. -# -# ---- LAUNCH IT (both supported) -------------------------------------------- -# 1) Single srun (SLURM provides the rank) — what run_xPyD_disagg.slurm does: -# srun --nodes=$((xP+yD)) --ntasks-per-node=1 \ -# bash .../vllm_disagg.sh node -# (IPADDRS must be ordered to match ranks: prefill IPs first, then decode, -# in the SAME order srun assigns PROCID — sort nodes alphabetically.) -# -# 2) Manual, one shell per node (rank from NODE_RANK; share LOG_PATH/FS): -# IPADDRS=ipP0,ipD0 xP=1 yD=1 NODE_RANK=0 bash vllm_disagg.sh node -# IPADDRS=ipP0,ipD0 xP=1 yD=1 NODE_RANK=1 bash vllm_disagg.sh node -# -# 3) Manual, fully granular (per-role): -# bash vllm_disagg.sh proxy -# bash vllm_disagg.sh prefill -# NODE_RANK=1 bash vllm_disagg.sh decode -# bash vllm_disagg.sh bench -# -# Cluster config: cluster.sh (sourced). Model flags: models.yaml. -# Parallelism: WIDE_EP_MODE=0 tp (independent TP servers) | 1 ep (DP+EP groups). -# ============================================================================= -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -_PROXY_IP_OVERRIDE="${PROXY_IP:-}" - -log() { echo "[vllm_disagg] $*"; } -die() { echo "ERROR: $*" >&2; exit 1; } - -usage() { - echo "Usage: $0 [node|proxy|prefill|decode|bench|accuracy] [--wide-ep-mode 0|1]" >&2 - echo " (no role => 'node': derive role from \$SLURM_PROCID/\$NODE_RANK)" >&2 - exit 1 -} - -# ----------------------------------------------------------------- arg parsing -# Sets globals: ROLE (default 'node'), _WIDE_EP_MODE_OVERRIDE. -parse_args() { - ROLE="${1:-node}" - shift || true - _WIDE_EP_MODE_OVERRIDE="" - while [[ $# -gt 0 ]]; do - case "$1" in - --wide-ep-mode) _WIDE_EP_MODE_OVERRIDE="${2:-}"; shift 2 ;; - --wide-ep-mode=*) _WIDE_EP_MODE_OVERRIDE="${1#*=}"; shift ;; - -h|--help) usage ;; - *) die "Unknown argument: $1" ;; - esac - done -} - -# ----------------------------------------------------------------- load config -# Sources cluster.sh and resolves WIDE_EP_MODE -> PARALLEL_MODE (tp|ep). -load_config() { - CLUSTER_ENV="${CLUSTER_ENV:-${SCRIPT_DIR}/cluster.sh}" - [[ -f "${CLUSTER_ENV}" ]] || die "cluster env file not found: ${CLUSTER_ENV}" - # shellcheck disable=SC1090 - source "${CLUSTER_ENV}" - - # Topology fanout normally arrives from cluster.sh; restate the defaults so - # the rank arithmetic is defined even when it does not. - : "${xP:=1}" "${yD:=1}" - - [[ -n "${_WIDE_EP_MODE_OVERRIDE}" ]] && WIDE_EP_MODE="${_WIDE_EP_MODE_OVERRIDE}" - WIDE_EP_MODE="${WIDE_EP_MODE:-0}" - case "${WIDE_EP_MODE}" in - 0) PARALLEL_MODE="tp" ;; - 1) PARALLEL_MODE="ep" ;; - *) die "WIDE_EP_MODE must be 0 (tp) or 1 (ep); got '${WIDE_EP_MODE}'" ;; - esac - - # node-mode orchestration knobs - RUN_AFTER_HEALTH="${RUN_AFTER_HEALTH:-accuracy}" # bench | accuracy | none - HEALTH_TIMEOUT_S="${HEALTH_TIMEOUT_S:-3600}" - - # MoRIIO KV transfer direction. 0 (default) - MORIIO_READ_MODE="${MORIIO_READ_MODE:-0}" -} - -# Emits the `read_mode` KV-config fragment (leading comma + newline) when read -# mode is enabled, else nothing (leaving MoRIIO's write-mode default). -moriio_read_mode_kv() { - case "${MORIIO_READ_MODE:-0}" in - 1|true|True|TRUE|yes|on) printf ',\n "read_mode": true' ;; - *) : ;; - esac -} - -# ----------------------------------------------------------------- topology -# Resolves MODEL_PATH, MODELS_YAML, the IP array, and the master/proxy addresses. -# IP_ARRAY[0..xP-1] = prefill nodes, [xP..xP+yD-1] = decode nodes. -resolve_topology() { - MODEL_PATH="${MODEL_DIR%/}/${MODEL_NAME}" - MODELS_YAML="${MODELS_YAML:-${SCRIPT_DIR}/models.yaml}" - mkdir -p "${LOG_PATH}" 2>/dev/null || true - - [[ -z "${IPADDRS}" ]] && IPADDRS="${PREFILL_IP},${DECODE_IP}" # 1P1D fallback - IFS=',' read -ra IP_ARRAY <<< "${IPADDRS}" - PREFILL_MASTER_ADDR="${IP_ARRAY[0]}" - DECODE_MASTER_ADDR="${IP_ARRAY[$xP]:-${IP_ARRAY[-1]}}" - # Proxy is co-located on the prefill master (rank 0) unless the CALLER - # explicitly set PROXY_IP. We use the pre-cluster.sh override (captured at the - # top) — not the live PROXY_IP, which cluster.sh always populates with the - # static PREFILL_IP fallback and would otherwise mask the real master. - PROXY_IP="${_PROXY_IP_OVERRIDE:-${PREFILL_MASTER_ADDR}}" -} - -# ============================================================================ -# Non-server roles -# ============================================================================ - -run_proxy() { - [[ -f "${PROXY_SCRIPT}" ]] || die "proxy script not found: ${PROXY_SCRIPT}" - log "proxy: http=:${PROXY_PORT} discovery=:${PROXY_PING_PORT} (${PROXY_SCRIPT})" - exec python3 "${PROXY_SCRIPT}" --port "${PROXY_PORT}" -} - -# Start the proxy in the BACKGROUND (node-mode rank 0). Sets PROXY_PID. -start_proxy_bg() { - [[ -f "${PROXY_SCRIPT}" ]] || die "proxy script not found: ${PROXY_SCRIPT}" - local plog="${LOG_PATH}/proxy_$(date +%Y%m%d_%H%M%S).log" - log "proxy(bg): http=:${PROXY_PORT} discovery=:${PROXY_PING_PORT} log=${plog}" - python3 "${PROXY_SCRIPT}" --port "${PROXY_PORT}" >"${plog}" 2>&1 & - PROXY_PID=$! -} - -run_bench() { - local base_url="http://127.0.0.1:${GATEWAY_PORT:-${PROXY_PORT}}" - local result_dir="${LOG_PATH}/bench_$(date +%Y%m%d_%H%M%S)" - mkdir -p "${result_dir}" - log "bench -> ${base_url} model=${MODEL_PATH} wide_ep_mode=${WIDE_EP_MODE}(${PARALLEL_MODE})" - log "combinations='${BENCHMARK_COMBINATIONS}' concurrency='${BENCHMARK_CON}'" - - local combo isl osl con nump logf - for combo in ${BENCHMARK_COMBINATIONS}; do - isl="${combo%%/*}"; osl="${combo##*/}" - for con in ${BENCHMARK_CON}; do - nump=$(( NUM_PROMPTS_FACTOR * con )) - (( nump < BENCHMARK_MIN_PROMPTS )) && nump="${BENCHMARK_MIN_PROMPTS}" - logf="${result_dir}/bench_${isl}_${osl}_con${con}.log" - log "ISL=${isl} OSL=${osl} CON=${con} num_prompts=${nump} -> ${logf}" - vllm bench serve \ - --backend openai \ - --base-url "${base_url}" \ - --endpoint /v1/completions \ - --model "${MODEL_PATH}" \ - --dataset-name random \ - --random-input-len "${isl}" \ - --random-output-len "${osl}" \ - --num-prompts "${nump}" \ - --max-concurrency "${con}" \ - --percentile-metrics ttft,tpot,itl,e2el \ - --ignore-eos \ - 2>&1 | tee "${logf}" - done - done - log "bench complete. Results in ${result_dir}" -} - -run_accuracy() { - # lm_eval isn't in the stock vLLM image; install on demand (rank 0 only, and - # only when accuracy is actually selected). Override with ACCURACY_PIP_SPEC. - if ! command -v lm_eval >/dev/null 2>&1; then - log "lm_eval not found — installing ${ACCURACY_PIP_SPEC:-lm_eval[api]}" - python3 -m pip install --no-cache-dir "${ACCURACY_PIP_SPEC:-lm_eval[api]}" - fi - local base_url="http://127.0.0.1:${GATEWAY_PORT:-${PROXY_PORT}}/v1/completions" - local ts; ts="$(date +%Y%m%d_%H%M%S)" - local logf="${LOG_PATH}/accuracy_${MODEL_NAME}_${PARALLEL_MODE}_${ts}.log" - local outdir="${LOG_PATH}/lm_eval_${MODEL_NAME}_${PARALLEL_MODE}_${ts}" - log "accuracy -> ${base_url} model=${MODEL_PATH} tasks=${ACCURACY_TASKS} wide_ep_mode=${WIDE_EP_MODE}(${PARALLEL_MODE})" - log "log=${logf} results=${outdir}" - - # --output_path makes lm_eval persist a results_*.json for scraping. - local eval_rc=0 - ( set -o pipefail - # Model weights are already served (loaded offline in a separate process); - # allow the Hub only here so lm_eval can fetch the eval dataset (e.g. gsm8k). - export HF_HUB_OFFLINE=0 HF_DATASETS_OFFLINE=0 - python3 -m lm_eval --model local-completions \ - --tasks "${ACCURACY_TASKS}" \ - --model_args "model=${MODEL_PATH},base_url=${base_url},num_concurrent=${ACCURACY_NUM_CONCURRENT},max_retries=${ACCURACY_MAX_RETRIES},tokenized_requests=False,trust_remote_code=True,timeout=${ACCURACY_TIMEOUT:-3600}" \ - --output_path "${outdir}" \ - 2>&1 | tee "${logf}" - ) || eval_rc=$? - if (( eval_rc != 0 )); then - log "FAIL: lm_eval exited rc=${eval_rc} (no usable score) — Log: ${logf}" - return "${eval_rc}" - fi - - # Gate on the score. Find the results JSON lm_eval just wrote, pull the max - # value of ACCURACY_METRIC across filters/tasks, and compare to the threshold. - local results_json="" - results_json="$(find "${outdir}" -name 'results_*.json' -type f 2>/dev/null | sort | tail -n1)" - if [[ -z "${results_json}" ]]; then - log "FAIL: no results_*.json under ${outdir} — cannot evaluate threshold" - return 1 - fi - - local score="" - score="$(python3 - "${results_json}" "${ACCURACY_METRIC}" <<'PY' -import json, sys -path, metric = sys.argv[1], sys.argv[2] -with open(path) as f: - data = json.load(f) -best = None -for task, metrics in (data.get("results") or {}).items(): - for key, val in metrics.items(): - if isinstance(val, bool) or not isinstance(val, (int, float)): - continue - if key.split(",")[0] == metric: - best = val if best is None else max(best, val) -print("" if best is None else "%.6f" % best) -PY -)" || score="" - - if [[ -z "${score}" ]]; then - log "FAIL: could not parse metric '${ACCURACY_METRIC}' from ${results_json}" - return 1 - fi - - if awk -v s="${score}" -v t="${ACCURACY_THRESHOLD}" 'BEGIN{exit !(s+0 >= t+0)}'; then - log "PASS: ${ACCURACY_METRIC}=${score} >= threshold=${ACCURACY_THRESHOLD} (tasks=${ACCURACY_TASKS}). Log: ${logf}" - return 0 - fi - log "FAIL: ${ACCURACY_METRIC}=${score} < threshold=${ACCURACY_THRESHOLD} (tasks=${ACCURACY_TASKS}). Log: ${logf}" - return 1 -} - -# ============================================================================ -# Server roles (prefill / decode) and their shared helpers -# ============================================================================ - -# Read model-specific flags from models.yaml (mode + role aware) into globals: -# MODEL_BASE_FLAGS / MODEL_ROLE_FLAGS / MODEL_EXPERIMENTAL_FLAGS -> MODEL_CONFIG -# Also exports the model's env: block (only if not already set; caller env wins). -load_model_flags() { - [[ -f "${MODELS_YAML}" ]] || die "models.yaml not found: ${MODELS_YAML}" - export MODELS_YAML MODEL_NAME PARALLEL_MODE ROLE - eval "$(python3 - <<'PY' -import os, shlex, sys, yaml -path = os.environ["MODELS_YAML"]; name = os.environ["MODEL_NAME"] -mode = os.environ["PARALLEL_MODE"]; role = os.environ["ROLE"] -with open(path, "r", encoding="utf-8") as f: - doc = yaml.safe_load(f) or {} -# Preferred form: top-level `models:` list of {model: , ...} entries. -# Back-compat: a top-level mapping keyed by model name. -entries = doc.get("models") if isinstance(doc, dict) else doc -cfg = None -if isinstance(entries, list): - cfg = next((e for e in entries if isinstance(e, dict) and e.get("model") == name), None) -elif isinstance(doc, dict): - cfg = doc.get(name) -if cfg is None: - print(f'echo "ERROR: model {name} not found in {path}" >&2; exit 1'); sys.exit(0) -role_cfg = cfg.get(role, {}) or {} -def q(v): return shlex.quote(str(v if v is not None else "")) -exports = { - "MODEL_BASE_FLAGS": cfg.get("base_flags", "") or "", - "MODEL_ROLE_FLAGS": role_cfg.get(mode, "") or "", - "MODEL_EXPERIMENTAL_FLAGS": cfg.get("experimental_flags", "") or "", -} -for k, v in exports.items(): print(f"{k}={q(v)}") -# Model-specific env: exported only if not already set (caller env wins). -for k, v in (cfg.get("env", {}) or {}).items(): - if k not in os.environ: - print(f"export {k}={q(v)}") -PY -)" - MODEL_CONFIG="${MODEL_BASE_FLAGS} ${MODEL_ROLE_FLAGS} ${MODEL_EXPERIMENTAL_FLAGS}" -} - -# Build the TP-mode (WIDE_EP_MODE=0) serve command into the global CMD array. -# Each node is an independent TP server registered to the proxy. -build_tp_cmd() { - local port="${TP_PORT}" - [[ -n "${HOST_IP}" ]] && CMD+=(--host "${HOST_IP}") - CMD+=(--port "${port}" --tensor-parallel-size "${TP_SIZE}") - local _mc; read -ra _mc <<< "${MODEL_CONFIG}"; CMD+=("${_mc[@]}") - local kv_cfg read_mode_kv - read_mode_kv="$(moriio_read_mode_kv)" - kv_cfg=$(cat < mori_high_throughput (paired with --enforce-eager in models.yaml) - # decode -> mori_low_latency (paired with CUDA graphs in models.yaml) - local default_backend="mori_low_latency" - [[ "${ROLE}" == "prefill" ]] && default_backend="mori_high_throughput" - local backend="${ALL2ALL_BACKEND:-${default_backend}}" - export VLLM_ALL2ALL_BACKEND="${VLLM_ALL2ALL_BACKEND:-${backend}}" - local port="${SERVE_PORT}" - CMD+=( - -tp 1 - --data-parallel-size "${DP_GROUP_SIZE}" - --data-parallel-size-local "${GPUS_PER_NODE}" - --data-parallel-address "${DP_MASTER_ADDR}" - --data-parallel-rpc-port "${RPC_PORT}" - --enable-expert-parallel - --all2all-backend "${backend}" - --port "${port}" - --no-enable-prefix-caching - --distributed-timeout-seconds "${DISTRIBUTED_TIMEOUT_SECONDS}" - ) - local _mc; read -ra _mc <<< "${MODEL_CONFIG}"; CMD+=("${_mc[@]}") - - if [[ "${IS_MASTER}" == "1" ]]; then - CMD+=(--api-server-count="${GPUS_PER_NODE}") - local kv_cfg read_mode_kv - read_mode_kv="$(moriio_read_mode_kv)" - kv_cfg=$(cat <&1 | tee "${LOGF}" -} - -run_prefill() { configure_prefill; run_server_fg; } -run_decode() { configure_decode; run_server_fg; } - -# ============================================================================ -# node mode — SLURM-native rank-based self-select + rank-0 orchestration -# ============================================================================ - -# Endpoints to health-check, one per line as "ip:port", depending on mode. -# tp: every prefill node (:PREFILL_PORT) and every decode node (:DECODE_PORT) -# ep: only the masters expose an API (prefill rank 0, decode rank xP) :SERVE_PORT -health_endpoints() { - local i - if [[ "${WIDE_EP_MODE}" == "0" ]]; then - for (( i=0; i/dev/null 2>&1; do - (( $(date +%s) >= deadline )) && { log "TIMEOUT waiting for ${ep}"; return 1; } - kill -0 "${SERVER_PID}" 2>/dev/null || { log "local server (pid ${SERVER_PID}) exited while waiting"; return 1; } - sleep 10 - done - log "healthy: ${ep}" - done - return 0 -} - -run_workload() { - case "${RUN_AFTER_HEALTH}" in - bench) run_bench ;; - accuracy) run_accuracy ;; - none|"") log "RUN_AFTER_HEALTH=none — skipping workload" ;; - *) die "unknown RUN_AFTER_HEALTH='${RUN_AFTER_HEALTH}'" ;; - esac -} - -# rank 0: proxy + health-gate + workload, then write the completion sentinel. -orchestrate_master() { - local sentinel="$1" rc=0 - # Front door: the proxy runs in-container (started here); the vllm-router - # runs as a SEPARATE container started by the SLURM job on this (rank-0) node, - # so in that mode we don't start anything here. - PROXY_PID="" - if [[ "${ROUTER_TYPE:-vllm-router}" == "vllm-router" ]]; then - log "ROUTER_TYPE=vllm-router: external router expected on gateway :${GATEWAY_PORT:-${ROUTER_PORT}} (not starting proxy)" - else - start_proxy_bg - fi - if wait_all_healthy; then - # If using the external vllm-router, wait for it to become ready before - # running the workload. The router starts as a separate container on this - # (rank-0) node; give it up to 300s to bind port ROUTER_PORT. - if [[ "${ROUTER_TYPE:-vllm-router}" == "vllm-router" ]]; then - local router_deadline=$(( $(date +%s) + 300 )) - log "waiting for vllm-router on :${ROUTER_PORT} (up to 300s)" - until /usr/bin/curl -sf "http://127.0.0.1:${ROUTER_PORT}/health" >/dev/null 2>&1; do - if (( $(date +%s) >= router_deadline )); then - log "FAIL: vllm-router not ready on :${ROUTER_PORT} after 300s (bring-up)" - rc=1 - break - fi - sleep 5 - done - (( rc == 0 )) && log "vllm-router healthy on :${ROUTER_PORT}" - fi - if (( rc == 0 )); then - set +e - run_workload - rc=$? - set -e - fi - else - log "FAIL: health-gate did not pass — server bring-up failed" - rc=1 - fi - echo "${rc}" > "${sentinel}" 2>/dev/null || true - log "master: workload rc=${rc}; sentinel=${sentinel}; tearing down local proxy+server" - [[ -n "${PROXY_PID:-}" ]] && kill "${PROXY_PID}" 2>/dev/null || true - kill "${SERVER_PID}" 2>/dev/null || true - wait "${SERVER_PID}" 2>/dev/null || true - return "${rc}" -} - -# non-master ranks: keep serving until the master signals completion (sentinel) -# or our local server dies unexpectedly. -watch_until_done() { - local sentinel="$1" rc=0 - log "rank ${NODE_RANK} serving; waiting for completion sentinel (${sentinel})" - while :; do - [[ -f "${sentinel}" ]] && { log "completion sentinel seen; shutting down"; break; } - if ! kill -0 "${SERVER_PID}" 2>/dev/null; then - log "local server (pid ${SERVER_PID}) exited before completion"; rc=1; break - fi - sleep 10 - done - kill "${SERVER_PID}" 2>/dev/null || true - wait "${SERVER_PID}" 2>/dev/null || true - return "${rc}" -} - -run_node() { - NODE_RANK="${NODE_RANK:-${SLURM_PROCID:-0}}" - local total=$(( xP + yD )) - (( NODE_RANK >= 0 && NODE_RANK < total )) \ - || die "NODE_RANK=${NODE_RANK} out of range [0,${total}) (xP=${xP} yD=${yD})" - - if (( NODE_RANK < xP )); then ROLE="prefill"; configure_prefill - else ROLE="decode"; configure_decode - fi - - # Shared-FS completion sentinel (LOG_PATH is shared & per-run/per-job). - local sentinel="${LOG_PATH}/.disagg_done" - - log "node mode: rank=${NODE_RANK}/${total} role=${ROLE} master=${IS_MASTER} wide_ep=${WIDE_EP_MODE}(${PARALLEL_MODE})" - log "topology: xP=${xP} yD=${yD} gpus/node=${GPUS_PER_NODE} IPADDRS=${IPADDRS}" - - # DRY_RUN: resolve+print the plan (role, serve cmd, endpoints) and exit 0. - # Validates wiring without launching vLLM / the proxy / any workload. - if [[ "${DRY_RUN:-0}" == "1" ]]; then - build_server_cmd - log "DRY_RUN: server cmd: ${CMD[*]}" - if (( NODE_RANK == 0 )); then - local eps; mapfile -t eps < <(health_endpoints) - log "DRY_RUN: health endpoints: ${eps[*]}" - log "DRY_RUN: post-health workload: ${RUN_AFTER_HEALTH}" - log "DRY_RUN: sentinel: ${sentinel}" - fi - log "DRY_RUN complete (rank ${NODE_RANK}) — no processes launched" - exit 0 - fi - - # aiter and flydsl keep runtime JIT/kernel caches inside the aiter install - # tree (…/aiter/jit and …/aiter/jit/flydsl_cache), which is read-only for the - # non-root (--user) container. AITER_JIT_DIR / FLYDSL_RUNTIME_CACHE_DIR - # redirect those to node-local /tmp (both are sanctioned overrides: aiter only - # points flydsl at its bundled cache when FLYDSL_RUNTIME_CACHE_DIR is unset). - # Seed each once from the image's bundled copy so we keep the prebuilt - # modules/kernels (fast, no NFS, no from-source rebuild / MLIR recompile). - local _aiter_src - _aiter_src="$(python3 -c 'import os,aiter; print(os.path.join(os.path.dirname(aiter.__file__),"jit"))' 2>/dev/null || true)" - if [[ -n "${AITER_JIT_DIR:-}" && ! -e "${AITER_JIT_DIR}/.seeded" ]]; then - if [[ -n "${_aiter_src}" && -d "${_aiter_src}" ]]; then - mkdir -p "${AITER_JIT_DIR}" - cp -a "${_aiter_src}/." "${AITER_JIT_DIR}/" 2>/dev/null || true - touch "${AITER_JIT_DIR}/.seeded" 2>/dev/null || true - log "seeded AITER_JIT_DIR=${AITER_JIT_DIR} from ${_aiter_src}" - else - log "WARN: could not locate aiter jit dir to seed AITER_JIT_DIR=${AITER_JIT_DIR}; aiter may rebuild from source" - fi - fi - if [[ -n "${FLYDSL_RUNTIME_CACHE_DIR:-}" && ! -e "${FLYDSL_RUNTIME_CACHE_DIR}/.seeded" ]]; then - mkdir -p "${FLYDSL_RUNTIME_CACHE_DIR}" - if [[ -n "${_aiter_src}" && -d "${_aiter_src}/flydsl_cache" ]]; then - cp -a "${_aiter_src}/flydsl_cache/." "${FLYDSL_RUNTIME_CACHE_DIR}/" 2>/dev/null || true - log "seeded FLYDSL_RUNTIME_CACHE_DIR=${FLYDSL_RUNTIME_CACHE_DIR} from ${_aiter_src}/flydsl_cache" - else - log "FLYDSL_RUNTIME_CACHE_DIR=${FLYDSL_RUNTIME_CACHE_DIR} set; no bundled flydsl_cache to seed (will JIT-compile at runtime)" - fi - touch "${FLYDSL_RUNTIME_CACHE_DIR}/.seeded" 2>/dev/null || true - fi - - # mori JIT-compiles its shmem/all2all kernels to ~/.mori/jit/_/ on - # first EP use. HOME here is /workspace, bind-mounted to persistent+shared NFS - # (/data/$USER), so a wrong-arch cache from an earlier run/build survives and - # gets reloaded even after MORI_GPU_ARCHS changes -- an existing cache dir - # wins over the arch setting. Reloading a gfx942 shmem_kernels.hsaco on gfx950 - # hardware fails with "device kernel image is invalid" and every EP worker - # dies at init (surfacing async as a torch.tensor HIP error). Guard: on EP - # runs, if the cache has no build for our target arch, purge it so mori - # recompiles for MORI_GPU_ARCHS. A matching arch dir is kept (fast reload). - if [[ "${WIDE_EP_MODE}" == "1" && -n "${MORI_GPU_ARCHS:-}" ]]; then - local _mori_jit="${HOME:-/workspace}/.mori/jit" - if [[ -d "${_mori_jit}" ]] && ! compgen -G "${_mori_jit}/${MORI_GPU_ARCHS}_*" >/dev/null 2>&1; then - log "purging stale mori jit cache ${_mori_jit} (no ${MORI_GPU_ARCHS}_* build present; forcing recompile for ${MORI_GPU_ARCHS})" - rm -rf "${_mori_jit}" 2>/dev/null || true - fi - fi - - # Start this node's server in the background (full server log -> LOGF; this - # shell's stdout stays free for orchestration logs the srun/CI captures). - build_server_cmd - "${CMD[@]}" >"${LOGF}" 2>&1 & - SERVER_PID=$! - log "server started pid=${SERVER_PID} log=${LOGF}" - - local rc=0 - if (( NODE_RANK == 0 )); then - orchestrate_master "${sentinel}" || rc=$? - # Canonical, human-readable end-of-run verdict for the login-node poller - # and CI logs (the .disagg_done sentinel remains the machine authority). - log "VERDICT: $( (( rc == 0 )) && echo PASS || echo FAIL) rc=${rc}" - else - watch_until_done "${sentinel}" || rc=$? - fi - log "node rank=${NODE_RANK} exiting rc=${rc}" - exit "${rc}" -} - -# ============================================================================ -main() { - parse_args "$@" - load_config - resolve_topology - case "${ROLE}" in - node) run_node ;; - proxy) run_proxy ;; - bench) run_bench ;; - accuracy) run_accuracy ;; - prefill) run_prefill ;; - decode) run_decode ;; - *) usage ;; - esac -} - -main "$@" diff --git a/.buildkite/check-torch-abi.py b/.buildkite/check-torch-abi.py deleted file mode 100644 index 5bfec8abd0..0000000000 --- a/.buildkite/check-torch-abi.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -"""Audit vLLM compiled libraries for PyTorch stable ABI compliance.""" - -import fnmatch -import sys -from pathlib import Path - -from torch_abi_audit import inspect_package -from torch_abi_audit.report import ExtensionReport, PackageReport - -# Temporary allowlist of extensions not yet on the stable ABI. -# Shrink and remove over time. -ALLOWED_UNSTABLE_LIBRARIES: tuple[str, ...] = ( - # ROCm-only legacy extensions. Most ROCm ops have moved to - # _C_stable_libtorch, but these two modules still bind libtorch directly. - "_C.abi3.so", - "_rocm_C.abi3.so", -) - - -def _relative_path(lib: ExtensionReport, package_root: Path) -> str: - try: - return lib.path.relative_to(package_root).as_posix() - except ValueError: - return lib.path.name - - -def _is_torch_unstable(lib: ExtensionReport) -> bool: - return lib.error is None and lib.torch.uses_torch and not lib.torch.stable - - -def _matches_allowlist(rel_path: str, patterns: tuple[str, ...]) -> bool: - return any(fnmatch.fnmatch(rel_path, pattern) for pattern in patterns) - - -def _iter_libs(report: PackageReport) -> tuple[ExtensionReport, ...]: - return (*report.extensions, *report.bundled_libs) - - -def _collect_unstable(report: PackageReport) -> list[str]: - return sorted( - _relative_path(lib, report.root) - for lib in _iter_libs(report) - if _is_torch_unstable(lib) - ) - - -def _find_stale_allowlist_entries( - report: PackageReport, patterns: tuple[str, ...] -) -> list[str]: - """Allowlist patterns that match a built library which is no longer unstable.""" - stale: list[str] = [] - for pattern in patterns: - for lib in _iter_libs(report): - if lib.error is not None: - continue - if not fnmatch.fnmatch(_relative_path(lib, report.root), pattern): - continue - if not _is_torch_unstable(lib): - stale.append(pattern) - break - return stale - - -def check_torch_abi( - package: str = "vllm", - patterns: tuple[str, ...] = ALLOWED_UNSTABLE_LIBRARIES, -) -> int: - report = inspect_package(package) - if report.error: - print(f"error: failed to inspect {package!r}: {report.error}", file=sys.stderr) - return 2 - - unstable = _collect_unstable(report) - unexpected = [ - rel_path for rel_path in unstable if not _matches_allowlist(rel_path, patterns) - ] - stale = _find_stale_allowlist_entries(report, patterns) - - if unexpected or stale: - if unexpected: - print( - "Not allowed: torch-unstable libraries outside " - f"ALLOWED_UNSTABLE_LIBRARIES: {', '.join(unexpected)}", - file=sys.stderr, - ) - if stale: - print( - "Not allowed: stale ALLOWED_UNSTABLE_LIBRARIES entries: " - f"{', '.join(stale)}", - file=sys.stderr, - ) - return 1 - - print("Torch stable ABI check passed.") - return 0 - - -if __name__ == "__main__": - print(">>> Auditing vLLM extension modules for PyTorch stable ABI compliance") - sys.exit(check_torch_abi()) diff --git a/.buildkite/check-wheel-size.py b/.buildkite/check-wheel-size.py deleted file mode 100644 index 77ee313687..0000000000 --- a/.buildkite/check-wheel-size.py +++ /dev/null @@ -1,53 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import os -import sys -import zipfile - -# Read the VLLM_MAX_SIZE_MB environment variable, defaulting to 500 MiB -# Note that we have 800 MiB quota, please use it wisely. -# See https://github.com/pypi/support/issues/6326 . -# Please also sync the value with the one in Dockerfile. -VLLM_MAX_SIZE_MB = int(os.environ.get("VLLM_MAX_SIZE_MB", 500)) - - -def print_top_10_largest_files(zip_file): - """Print the top 10 largest files in the given zip file.""" - with zipfile.ZipFile(zip_file, "r") as z: - file_sizes = [(f, z.getinfo(f).file_size) for f in z.namelist()] - file_sizes.sort(key=lambda x: x[1], reverse=True) - for f, size in file_sizes[:10]: - print(f"{f}: {size / (1024 * 1024):.2f} MBs uncompressed.") - - -def check_wheel_size(directory): - """Check the size of .whl files in the given directory.""" - for root, _, files in os.walk(directory): - for file_name in files: - if file_name.endswith(".whl"): - wheel_path = os.path.join(root, file_name) - wheel_size_mb = os.path.getsize(wheel_path) / (1024 * 1024) - if wheel_size_mb > VLLM_MAX_SIZE_MB: - print( - f"Not allowed: Wheel {wheel_path} is larger " - f"({wheel_size_mb:.2f} MB) than the limit " - f"({VLLM_MAX_SIZE_MB} MB)." - ) - print_top_10_largest_files(wheel_path) - return 1 - else: - print( - f"Wheel {wheel_path} is within the allowed size " - f"({wheel_size_mb:.2f} MB)." - ) - return 0 - - -if __name__ == "__main__": - if len(sys.argv) < 2: - print("Usage: python check-wheel-size.py ") - sys.exit(1) - - directory = sys.argv[1] - sys.exit(check_wheel_size(directory)) diff --git a/.buildkite/ci_config.yaml b/.buildkite/ci_config.yaml deleted file mode 100644 index 9e1e46db67..0000000000 --- a/.buildkite/ci_config.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: vllm_ci -job_dirs: - - ".buildkite/image_build" - - ".buildkite/test_areas" - - ".buildkite/hardware_tests" -run_all_patterns: - - "docker/Dockerfile" - - "CMakeLists.txt" - - "requirements/common.txt" - - "requirements/cuda.txt" - - "requirements/kv_connectors.txt" - - "requirements/build/cuda.txt" - - "requirements/test/cuda.txt" - - "setup.py" - - "csrc/" - - "cmake/" - - ".buildkite/check-torch-abi.py" -run_all_exclude_patterns: - - "docker/Dockerfile." - - "csrc/cpu/" - - "csrc/rocm/" - - "cmake/hipify.py" - - "cmake/cpu_extension.cmake" -registries: public.ecr.aws/q9t5s3a7 -repositories: - main: "vllm-ci-postmerge-repo" - premerge: "vllm-ci-test-repo" diff --git a/.buildkite/ci_config_intel.yaml b/.buildkite/ci_config_intel.yaml deleted file mode 100644 index cd6bec005d..0000000000 --- a/.buildkite/ci_config_intel.yaml +++ /dev/null @@ -1,23 +0,0 @@ -name: vllm_intel_ci -job_dirs: - - ".buildkite/intel_jobs" -run_all_patterns: - - ".buildkite/ci_config_intel.yaml" - - ".buildkite/scripts/hardware_ci/run-intel-test.sh" - - "docker/Dockerfile" - - "docker/Dockerfile.xpu" - - "CMakeLists.txt" - - "requirements/common.txt" - - "requirements/xpu.txt" - - "setup.py" - - "csrc/" - - "cmake/" -run_all_exclude_patterns: - - "csrc/cpu/" - - "csrc/rocm/" - - "cmake/hipify.py" - - "cmake/cpu_extension.cmake" -registries: public.ecr.aws/q9t5s3a7 -repositories: - main: "vllm-ci-test-repo" - premerge: "vllm-ci-test-repo" diff --git a/.buildkite/ci_config_rocm.yaml b/.buildkite/ci_config_rocm.yaml deleted file mode 100644 index baeb1af145..0000000000 --- a/.buildkite/ci_config_rocm.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: vllm_rocm_ci -job_dirs: - - ".buildkite/hardware_tests" -run_all_patterns: - - ".dockerignore" - - "docker/Dockerfile.rocm" - - "docker/Dockerfile.rocm_base" - - "docker/ci-rocm.hcl" - - "docker/docker-bake-rocm.hcl" - - ".buildkite/hardware_tests/amd.yaml" - - ".buildkite/ci_config_rocm.yaml" - - ".buildkite/scripts/ci-bake-rocm.sh" - - ".buildkite/scripts/rocm/" - - ".buildkite/scripts/hardware_ci/run-amd-test.py" - - ".buildkite/scripts/hardware_ci/run-amd-test.sh" - - "CMakeLists.txt" - - "requirements/common.txt" - - "requirements/rocm.txt" - - "requirements/build/rust.txt" - - "requirements/build/rocm.txt" - - "requirements/test/rocm.txt" - - "pyproject.toml" - - "setup.py" - - "tools/build_rust.sh" - - "rust/" - - "rust-toolchain.toml" - - "tools/build_rust.py" - - "tools/install_torchcodec_rocm.sh" - - "tests/vllm_test_utils/" - - "vllm/envs.py" - - "vllm/__init__.py" - - "csrc/" - - "cmake/" -run_all_exclude_patterns: - - "csrc/cpu/" - - "cmake/cpu_extension.cmake" diff --git a/.buildkite/hardware_tests/amd.yaml b/.buildkite/hardware_tests/amd.yaml deleted file mode 100644 index 66d8a589ab..0000000000 --- a/.buildkite/hardware_tests/amd.yaml +++ /dev/null @@ -1,88 +0,0 @@ -group: Hardware - AMD Build - -# ROCm image flow: -# 1. Select the content-matching ROCm base, building only on a cache miss. -# 2. Build ci_base from either the stable base or the newly selected base. -# 3. Build the build-scoped ROCm CI image and smoke-test it before GPU jobs run. -steps: - - label: "AMD: :docker: refresh ROCm base" - key: refresh-rocm-base-amd - depends_on: [] - device: amd_cpu - no_plugin: true - commands: - - bash .buildkite/scripts/rocm/refresh-base-image.sh - concurrency: 1 - concurrency_group: "vllm/rocm/base-build/$BUILDKITE_COMMIT" - env: - DOCKER_BUILDKIT: "1" - BUILDKIT_PROGRESS: "tty" - TERM: "xterm-256color" - retry: - automatic: - - exit_status: 1 # Transient Docker/BuildKit failure - limit: 1 - - exit_status: -1 # Agent was lost - limit: 1 - - exit_status: -10 # Agent was lost - limit: 1 - - # Ensure ci_base is up-to-date before building the test image. - # Compares a content hash of ci_base-affecting files against the remote - # image label. If hashes match the build is skipped (< 30 s); if they - # differ ci_base is rebuilt and pushed automatically. - - label: "AMD: :docker: ensure ci_base" - key: ensure-ci-base-amd - soft_fail: false - depends_on: - - refresh-rocm-base-amd - device: amd_cpu - no_plugin: true - commands: - - bash .buildkite/scripts/rocm/build-ci-base.sh - concurrency: 1 - concurrency_group: "vllm/rocm/ci-base-build/$BUILDKITE_COMMIT" - env: - DOCKER_BUILDKIT: "1" - BUILDKIT_PROGRESS: "tty" - TERM: "xterm-256color" - VLLM_BAKE_FILE: "docker/docker-bake-rocm.hcl" - PYTORCH_ROCM_ARCH: "gfx90a;gfx942;gfx950" - REMOTE_VLLM: "0" - retry: - automatic: - - exit_status: 1 # Transient Docker/BuildKit failure - limit: 1 - - exit_status: 2 # Exact image found but registry handoff failed - limit: 1 - - exit_status: -1 # Agent was lost - limit: 1 - - exit_status: -10 # Agent was lost - limit: 1 - - - label: "AMD: :docker: build test image and artifacts" - key: image-build-amd - soft_fail: false - depends_on: - - ensure-ci-base-amd - device: amd_cpu - no_plugin: true - commands: - - bash .buildkite/scripts/rocm/build-test-image.sh - - bash .buildkite/scripts/rocm/smoke-test-image.sh - env: - DOCKER_BUILDKIT: "1" - BUILDKIT_PROGRESS: "tty" - TERM: "xterm-256color" - VLLM_BAKE_FILE: "docker/docker-bake-rocm.hcl" - PYTORCH_ROCM_ARCH: "gfx90a;gfx942;gfx950" - IMAGE_TAG: "rocm/vllm-ci:build-$BUILDKITE_BUILD_ID" - REMOTE_VLLM: "0" - retry: - automatic: - - exit_status: 1 # Transient Docker/BuildKit failure - limit: 1 - - exit_status: -1 # Agent was lost - limit: 1 - - exit_status: -10 # Agent was lost - limit: 1 diff --git a/.buildkite/hardware_tests/ascend_npu.yaml b/.buildkite/hardware_tests/ascend_npu.yaml deleted file mode 100644 index 80aa1eb38d..0000000000 --- a/.buildkite/hardware_tests/ascend_npu.yaml +++ /dev/null @@ -1,104 +0,0 @@ -group: Hardware -depends_on: ~ -steps: - - label: "Ascend NPU Test" - key: ascend-npu-test - soft_fail: true - no_plugin: true - device: ascend_npu - timeout_in_minutes: 30 - source_file_dependencies: - - vllm/ - - .buildkite/scripts/hardware_ci/run-npu-test.sh - - "!vllm/platforms/cuda.py" - - "!vllm/platforms/xpu.py" - - "!vllm/platforms/tpu.py" - - "!vllm/platforms/cpu.py" - - "!vllm/platforms/zen_cpu.py" - - "!vllm/cute_utils/" - - "!vllm/tilelang_utils/" - - "!vllm/device_allocator/" - - "!vllm/vllm_flash_attn/" - - "!vllm/kernels/" - - "!vllm/model_executor/kernels/" - - "!vllm/engine/" - - "!vllm/entrypoints/" - - "!vllm/ray/" - - "!vllm/v1/attention/backends/flash_attn.py" - - "!vllm/v1/attention/backends/flash_attn_diffkv.py" - - "!vllm/v1/attention/backends/flashinfer.py" - - "!vllm/v1/attention/backends/triton_attn.py" - - "!vllm/v1/attention/backends/triton_attn_diffkv.py" - - "!vllm/v1/attention/backends/rocm_attn.py" - - "!vllm/v1/attention/backends/rocm_aiter_fa.py" - - "!vllm/v1/attention/backends/rocm_aiter_unified_attn.py" - - "!vllm/v1/attention/backends/flex_attention.py" - - "!vllm/v1/attention/backends/hpc_attn.py" - - "!vllm/v1/attention/backends/mamba1_attn.py" - - "!vllm/v1/attention/backends/mamba2_attn.py" - - "!vllm/v1/attention/backends/mamba_attn.py" - - "!vllm/v1/attention/backends/short_conv_attn.py" - - "!vllm/v1/attention/backends/turboquant_attn.py" - - "!vllm/v1/attention/backends/cpu_attn.py" - - "!vllm/v1/attention/backends/fa_utils.py" - - "!vllm/distributed/device_communicators/pynccl.py" - - "!vllm/distributed/device_communicators/pynccl_wrapper.py" - - "!vllm/distributed/device_communicators/pynccl_allocator.py" - - "!vllm/distributed/device_communicators/xpu_communicator.py" - - "!vllm/distributed/device_communicators/cpu_communicator.py" - - "!vllm/distributed/device_communicators/cuda_wrapper.py" - - "!vllm/distributed/device_communicators/flashinfer_all_reduce.py" - - "!vllm/distributed/device_communicators/aiter_custom_all_reduce.py" - - "!vllm/distributed/device_communicators/custom_all_reduce.py" - - "!vllm/distributed/device_communicators/quick_all_reduce.py" - - "!vllm/distributed/device_communicators/all2all.py" - - "!vllm/distributed/device_communicators/all_reduce_utils.py" - - "!vllm/distributed/device_communicators/shm_object_storage.py" - - "!vllm/distributed/device_communicators/symm_mem.py" - - "!vllm/distributed/device_communicators/mnnvl_compat.py" - - "!vllm/third_party/flashmla/" - - "!vllm/third_party/pynvml.py" - - "!vllm/_custom_ops.py" - - "!vllm/_xpu_ops.py" - - "!vllm/_aiter_ops" - - "!vllm/collect_env.py" - - "!vllm/connections.py" - - "!vllm/env_override.py" - - "!vllm/logits_process.py" - - "!vllm/logprobs.py" - - "!vllm/model_inspection.py" - - "!vllm/outputs.py" - - "!vllm/scalar_type.py" - - "!vllm/scripts.py" - - "!vllm/version.py" - - "!vllm/assets/" - - "!vllm/benchmarks/" - - "!vllm/inputs/" - - "!vllm/ir/" - - "!vllm/parser/" - - "!vllm/plugins/" - - "!vllm/reasoning/" - - "!vllm/renderers/" - - "!vllm/tokenizers/" - - "!vllm/tool_parsers/" - - "!vllm/tracing/" - - "!vllm/usage/" - - "!tests/" - - "!benchmarks/" - - "!docs/" - - "!examples/" - - "!scripts/" - - "!vllm/model_executor/layers/quantization/auto_awq.py" - - "!vllm/model_executor/layers/quantization/auto_gptq.py" - - "!vllm/model_executor/layers/quantization/inc/" - - "!vllm/model_executor/layers/quantization/awq_triton.py" - - "!vllm/model_executor/layers/quantization/experts_int8.py" - - "!vllm/model_executor/layers/quantization/fbgemm_fp8.py" - - "!vllm/model_executor/layers/quantization/fp_quant.py" - - "!vllm/model_executor/layers/quantization/humming.py" - - "!vllm/model_executor/layers/quantization/moe_wna16.py" - - "!vllm/model_executor/layers/quantization/qutlass_utils.py" - - "!vllm/model_executor/layers/quantization/quark/" - - "!vllm/model_executor/layers/quantization/turboquant/" - commands: - - bash .buildkite/scripts/hardware_ci/run-npu-test.sh diff --git a/.buildkite/hardware_tests/cpu.yaml b/.buildkite/hardware_tests/cpu.yaml deleted file mode 100644 index 64d5a654f5..0000000000 --- a/.buildkite/hardware_tests/cpu.yaml +++ /dev/null @@ -1,288 +0,0 @@ -group: CPU -depends_on: [] -steps: -- label: CPU-Kernel Tests Shard %N - key: cpu-kernel-tests - depends_on: [] - device: intel_cpu - no_plugin: true - source_file_dependencies: - - csrc/cpu/ - - cmake/cpu_extension.cmake - - CMakeLists.txt - - vllm/_custom_ops.py - - tests/kernels/attention/test_cpu_attn.py - - tests/v1/attention/test_group_head_counts.py - - tests/kernels/attention/test_amx_mla.py - - vllm/v1/attention/backends/mla/amx_mla.py - - vllm/model_executor/layers/attention/mla_attention.py - - tests/kernels/test_deepseek_v4_cpu_kernels.py - - tests/kernels/moe/test_cpu_fused_moe.py - - tests/kernels/moe/test_cpu_quant_fused_moe.py - - tests/kernels/moe/test_zen_cpu_int8_moe.py - - tests/test_zentorch_version_pin.py - - vllm/model_executor/layers/fused_moe/experts/cpu_moe.py - - tests/kernels/test_onednn.py - - tests/kernels/test_cpu_fused_sampling.py - - tests/kernels/test_awq_int4_to_int8.py - - tests/kernels/quantization/test_cpu_fp8_scaled_mm.py - - tests/kernels/quantization/test_zen_da8w4.py - - vllm/model_executor/kernels/linear/mixed_precision/zentorch.py - - vllm/model_executor/layers/fused_moe/experts/zentorch_moe.py - - tests/kernels/mamba/cpu/test_cpu_gdn_ops.py - - tests/kernels/mamba/test_cpu_short_conv.py - - tests/kernels/mamba/test_causal_conv1d.py - - tests/kernels/mamba/test_mamba_ssm.py - - vllm/v1/sample/ops/topk_topp_triton.py - - vllm/v1/sample/ops/topk_topp_sampler.py - - tests/v1/sample/test_topk_topp_sampler.py - commands: - - | - bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m " - pytest -x -v -s --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB \ - tests/kernels/attention/test_cpu_attn.py \ - tests/v1/attention/test_group_head_counts.py \ - tests/kernels/attention/test_amx_mla.py \ - tests/kernels/test_deepseek_v4_cpu_kernels.py \ - tests/kernels/moe/test_cpu_fused_moe.py \ - tests/kernels/moe/test_cpu_quant_fused_moe.py \ - tests/kernels/moe/test_zen_cpu_int8_moe.py \ - tests/test_zentorch_version_pin.py \ - tests/kernels/mamba/test_cpu_short_conv.py \ - tests/kernels/test_onednn.py \ - tests/kernels/test_cpu_fused_sampling.py \ - tests/kernels/test_awq_int4_to_int8.py \ - tests/kernels/quantization/test_cpu_fp8_scaled_mm.py \ - tests/kernels/quantization/test_zen_da8w4.py \ - tests/kernels/mamba/cpu/test_cpu_gdn_ops.py \ - tests/kernels/mamba/test_causal_conv1d.py \ - tests/kernels/mamba/test_mamba_ssm.py \ - tests/v1/sample/test_topk_topp_sampler.py::TestTritonTopkTopp" - parallelism: 2 - -# Speculative decoding under a CPU platform. The unit tests already pass on -# CPU but no CPU job runs them; the e2e tests gate the engine path itself, -# which unit tests cannot reach. -- label: CPU-Spec Decode Tests - key: cpu-spec-decode-tests - depends_on: [] - soft_fail: true - device: intel_cpu - no_plugin: true - source_file_dependencies: - - vllm/v1/spec_decode/ - - vllm/v1/worker/cpu/ - - vllm/v1/worker/cpu_model_runner.py - - vllm/v1/worker/gpu/spec_decode/ - - vllm/v1/attention/backends/cpu_attn.py - - vllm/config/speculative.py - - vllm/model_executor/models/qwen3_5.py - - vllm/model_executor/models/qwen3_5_mtp.py - - vllm/model_executor/models/llama_eagle3.py - - vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py - - vllm/model_executor/layers/mamba/mamba_mixer2.py - - vllm/model_executor/layers/mamba/ops/cpu/ - - tests/v1/e2e/test_cpu_spec_decode.py - - tests/v1/spec_decode/ - - tests/v1/worker/ - commands: - - | - bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m " - pytest -x -v -s \ - tests/v1/worker/test_gpu_target_dependent_ar_speculator.py \ - tests/v1/worker/test_gpu_extract_hidden_states_speculator.py \ - tests/v1/worker/test_gpu_batch_ordering.py \ - tests/v1/worker/test_gpu_rejection_sampler_chunking.py \ - tests/v1/worker/test_eagle3_aux_hidden_states_pp.py \ - tests/v1/worker/test_spec_decode_embed_sharing_pp.py \ - tests/v1/spec_decode/test_adaptive_verification.py \ - tests/v1/spec_decode/test_dflash2.py \ - tests/v1/spec_decode/test_dflash_causality.py \ - tests/v1/spec_decode/test_eagle_draft_attn_metadata.py \ - tests/v1/spec_decode/test_draft_attention_backend_override.py \ - tests/v1/spec_decode/test_draft_moe_backend_override.py \ - tests/v1/spec_decode/test_ngram.py \ - tests/v1/spec_decode/test_llm_base_proposer_sampling.py - unset VLLM_CPU_CI_ENV - pytest -x -v -s tests/v1/e2e/test_cpu_spec_decode.py" - -# Note: SDE can't be downloaded from CI host because of AWS WAF -# - label: CPU-Compatibility Tests -# depends_on: [] -# device: intel_cpu -# no_plugin: true -# source_file_dependencies: -# - cmake/cpu_extension.cmake -# - setup.py -# - vllm/platforms/cpu.py -# commands: -# - | -# bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 20m " -# bash .buildkite/scripts/hardware_ci/run-cpu-compatibility-test.sh" - -- label: CPU-Language Generation and Pooling Model Tests Shard %N - key: cpu-language-generation-and-pooling-model-tests - depends_on: [] - device: intel_cpu - no_plugin: true - source_file_dependencies: - - csrc/cpu/ - - vllm/ - - tests/models/language/generation/ - - tests/models/language/pooling/ - - tests/v1/e2e/test_cpu_linear_attn_chunked_prefix.py - commands: - - | - bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 25m " - pytest -x -v -s tests/models/language/generation tests/models/language/pooling tests/v1/e2e/test_cpu_linear_attn_chunked_prefix.py -m cpu_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB" - parallelism: 3 - -- label: CPU-Quantization Model Tests - key: cpu-quantization-model-tests - depends_on: [] - device: intel_cpu - no_plugin: true - source_file_dependencies: - - csrc/cpu/ - - vllm/model_executor/layers/quantization/auto_gptq.py - - vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py - - vllm/model_executor/kernels/linear/mixed_precision/cpu.py - - vllm/model_executor/kernels/linear/scaled_mm/cpu.py - - vllm/model_executor/layers/fused_moe/experts/cpu_moe.py - - tests/quantization/test_compressed_tensors.py - - tests/quantization/test_cpu_wna16.py - - tests/quantization/test_cpu_w8a8.py - commands: - - | - bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m " - pytest -x -v -s tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_logprobs - pytest -x -v -s tests/quantization/test_cpu_wna16.py - pytest -x -v -s tests/quantization/test_cpu_w8a8.py" - -- label: CPU-Distributed Tests (PP+TP) - key: cpu-distributed-tests-pp-tp - depends_on: [] - device: intel_cpu - no_plugin: true - source_file_dependencies: &cpu_distributed_deps - - csrc/cpu/shm.cpp - - vllm/v1/worker/cpu_worker.py - - vllm/v1/worker/gpu_worker.py - - vllm/v1/worker/cpu_model_runner.py - - vllm/v1/worker/gpu_model_runner.py - - vllm/platforms/cpu.py - - vllm/distributed/parallel_state.py - - vllm/distributed/device_communicators/cpu_communicator.py - - .buildkite/scripts/hardware_ci/run-cpu-distributed-smoke-test.sh - commands: - - | - bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 10m " - bash .buildkite/scripts/hardware_ci/run-cpu-distributed-smoke-test.sh tp_pp" - -- label: CPU-Distributed Tests (DP+TP) - key: cpu-distributed-tests-dp-tp - depends_on: [] - device: intel_cpu - no_plugin: true - source_file_dependencies: *cpu_distributed_deps - commands: - - | - bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 10m " - bash .buildkite/scripts/hardware_ci/run-cpu-distributed-smoke-test.sh dp_tp" - -- label: CPU-Multi-Modal Model Tests %N - key: cpu-multi-modal-model-tests-n - depends_on: [] - device: intel_cpu - no_plugin: true - source_file_dependencies: - # - vllm/ - - vllm/model_executor/layers/rotary_embedding - - tests/models/multimodal/generation/ - commands: - - | - bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m " - pytest -x -v -s tests/models/multimodal/generation --ignore=tests/models/multimodal/generation/test_pixtral.py --ignore=tests/models/multimodal/generation/test_qwen2_5_vl.py -m cpu_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB" - parallelism: 4 - -- label: CPU-Qwen2.5-VL Multimodal Tests - key: cpu-qwen2-5-vl-multimodal-tests - depends_on: [] - device: intel_cpu - no_plugin: true - source_file_dependencies: - # - vllm/ - - vllm/model_executor/layers/rotary_embedding - - tests/models/multimodal/generation/ - commands: - - | - bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 40m " - VLLM_CI_ENV=0 pytest -x -v -s tests/models/multimodal/generation/test_qwen2_5_vl.py" - -- label: "Arm CPU Test Shard %N" - key: arm-cpu-test - depends_on: - - cpu-arm64-image-build - soft_fail: false - device: arm_cpu - no_plugin: true - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-cpu-test-arm.sh - "$$BUILDKITE_PARALLEL_JOB" - "$REGISTRY/$REPO:$BUILDKITE_COMMIT-arm64-cpu" - parallelism: 3 - -- label: ":docker: Build Zen CPU image" - key: zen-cpu-image-build - # Reuse the shared `:-cpu` image that image-build-cpu publishes - # to public ECR, so Dockerfile.zen just layers on top instead of recompiling - # vLLM from source. If the pull fails, the build script still falls back to - # building the CPU base from source. - depends_on: - - image-build-cpu - soft_fail: true - agents: - queue: amd-cpu - source_file_dependencies: - - .buildkite/hardware_tests/cpu.yaml - - .buildkite/image_build/image_build_zen_cpu.sh - - docker/Dockerfile.zen - - docker/Dockerfile.cpu - - requirements/cpu.txt - - requirements/build/cpu.txt - - requirements/test/cpu.txt - - cmake/cpu_extension.cmake - - vllm/model_executor/layers/utils.py - - vllm/model_executor/kernels/linear/zentorch_utils.py - - vllm/model_executor/kernels/linear/scaled_mm/zentorch.py - - vllm/model_executor/kernels/linear/mixed_precision/zentorch.py - - vllm/platforms/cpu.py - - vllm/platforms/zen_cpu.py - - vllm/platforms/__init__.py - - tests/model_executor/test_cpu_unquantized_gemm_dispatch.py - - tests/test_zen_cpu_platform_detection.py - - tests/kernels/moe/test_zen_cpu_fused_moe.py - - tests/kernels/moe/test_zen_cpu_int8_moe.py - - vllm/v1/attention/backends/zentorch_sdpa.py - - tests/test_zentorch_version_pin.py - - vllm/v1/attention/backends/mla/prefill/zen_cpu_sdpa.py - - vllm/model_executor/layers/fused_moe/experts/zentorch_moe.py - - tests/kernels/quantization/test_zen_da8w4.py - commands: - - .buildkite/image_build/image_build_zen_cpu.sh "$REGISTRY" "$REPO" "$BUILDKITE_COMMIT" - env: - DOCKER_BUILDKIT: "1" - # Install zentorch directly (if-branch in Dockerfile.zen) instead of - # `uv pip install "vllm[zen]"`, which forces uv to re-resolve vLLM's full - # dependency closure and chokes on its transitive triton-cpu URL dependency. - # Keep this pin in sync with the `zen` extra in setup.py. - ZENTORCH_VERSION: "2.14.0.1" - retry: - automatic: - - exit_status: 1 # Transient Docker/BuildKit failure - limit: 1 - - exit_status: -1 # Agent was lost - limit: 2 - - exit_status: -10 # Agent was lost - limit: 2 diff --git a/.buildkite/hardware_tests/gh200.yaml b/.buildkite/hardware_tests/gh200.yaml deleted file mode 100644 index 54f2e27632..0000000000 --- a/.buildkite/hardware_tests/gh200.yaml +++ /dev/null @@ -1,16 +0,0 @@ -group: Hardware -steps: - - label: "GH200 Test" - key: gh200-test - timeout_in_minutes: 30 - soft_fail: true - device: gh200 - optional: true - working_dir: "/vllm-workspace" - depends_on: - - arm64-image-build - env: - VLLM_WORKER_MULTIPROC_METHOD: "spawn" - commands: - - nvidia-smi - - python3 examples/basic/offline_inference/generate.py --model meta-llama/Llama-3.2-1B diff --git a/.buildkite/hardware_tests/intel.yaml b/.buildkite/hardware_tests/intel.yaml deleted file mode 100644 index 40ead86838..0000000000 --- a/.buildkite/hardware_tests/intel.yaml +++ /dev/null @@ -1,11 +0,0 @@ -group: Hardware -depends_on: ~ -steps: - - label: "Intel HPU Test" - key: intel-hpu-test - soft_fail: true - device: intel_hpu - no_plugin: true - commands: - - bash .buildkite/scripts/hardware_ci/run-hpu-test.sh - diff --git a/.buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml b/.buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml deleted file mode 100644 index ed98d44085..0000000000 --- a/.buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml +++ /dev/null @@ -1,175 +0,0 @@ -group: Intel -steps: - - label: ":docker: Build XPU image" - soft_fail: true - optional: true - depends_on: [] - key: image-build-xpu - commands: - - bash -lc '.buildkite/image_build/image_build_xpu.sh "public.ecr.aws/q9t5s3a7" "vllm-ci-test-repo" "$BUILDKITE_COMMIT"' - env: - DOCKER_BUILDKIT: "1" - retry: - automatic: - - exit_status: -1 # Agent was lost - limit: 2 - - exit_status: -10 # Agent was lost - limit: 2 - - label: ":intel: (BMG) Example Test" - key: xpu-example-test - depends_on: - - image-build-xpu - timeout_in_minutes: 50 - optional: true - device: intel_gpu - agent_tags: - label: production - gpu: 2+ - mem: 24+ - no_plugin: true - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - source_file_dependencies: - - .buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml - - .buildkite/scripts/hardware_ci/run-intel-ci-test.sh - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh example' - - label: ":intel: (BMG) V1 test" - key: xpu-v1-test - depends_on: - - image-build-xpu - timeout_in_minutes: 70 - optional: true - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 24+ - no_plugin: true - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - source_file_dependencies: - - .buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml - - .buildkite/scripts/hardware_ci/run-intel-ci-test.sh - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh v1' - parallelism: 2 - - label: ":intel: (BMG) W8A8 FP8 Linear Examples" - key: xpu-w8a8-fp8-linear-examples - depends_on: - - image-build-xpu - timeout_in_minutes: 60 - optional: true - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 24+ - no_plugin: true - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - .buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml - - .buildkite/scripts/hardware_ci/run-intel-ci-test.sh - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh w8a8-fp8-linear' - - label: ":intel: (BMG) Server test" - key: xpu-server-test - depends_on: - - image-build-xpu - timeout_in_minutes: 45 - optional: true - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - source_file_dependencies: - - .buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml - - .buildkite/scripts/hardware_ci/run-intel-ci-test.sh - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh server' - - label: ":intel: (BMG) Quantization test" - key: xpu-quantization-test - depends_on: - - image-build-xpu - timeout_in_minutes: 60 - optional: true - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - source_file_dependencies: - - .buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml - - .buildkite/scripts/hardware_ci/run-intel-ci-test.sh - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh quantization' - - label: ":intel: (BMG) Compressed tensors FP8 test" - key: xpu-compressed-tensors-fp8-test - depends_on: - - image-build-xpu - timeout_in_minutes: 60 - optional: true - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - .buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml - - .buildkite/scripts/hardware_ci/run-intel-ci-test.sh - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh compressed-tensors-fp8' - - label: ":intel: (BMG) Graph" - key: xpu-graph - depends_on: - - image-build-xpu - timeout_in_minutes: 60 - optional: true - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 32+ - no_plugin: true - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - .buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml - - .buildkite/scripts/hardware_ci/run-intel-ci-test.sh - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh graph' diff --git a/.buildkite/image_build/image_build.sh b/.buildkite/image_build/image_build.sh deleted file mode 100755 index 6d22acd5b1..0000000000 --- a/.buildkite/image_build/image_build.sh +++ /dev/null @@ -1,376 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# replace invalid characters in Docker image tags and truncate to 128 chars -clean_docker_tag() { - local input="$1" - echo "$input" | sed 's/[^a-zA-Z0-9._-]/_/g' | cut -c1-128 -} - -print_usage_and_exit() { - echo "Usage: $0 []" - exit 1 -} - -print_instance_info() { - echo "" - echo "=== Debug: Instance Information ===" - # Get IMDSv2 token - if TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" \ - -H "X-aws-ec2-metadata-token-ttl-seconds: 21600" 2>/dev/null); then - AMI_ID=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \ - http://169.254.169.254/latest/meta-data/ami-id 2>/dev/null || echo "unknown") - INSTANCE_TYPE=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \ - http://169.254.169.254/latest/meta-data/instance-type 2>/dev/null || echo "unknown") - INSTANCE_ID=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \ - http://169.254.169.254/latest/meta-data/instance-id 2>/dev/null || echo "unknown") - AZ=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \ - http://169.254.169.254/latest/meta-data/placement/availability-zone 2>/dev/null || echo "unknown") - echo "AMI ID: ${AMI_ID}" - echo "Instance Type: ${INSTANCE_TYPE}" - echo "Instance ID: ${INSTANCE_ID}" - echo "AZ: ${AZ}" - else - echo "Not running on EC2 or IMDS not available" - fi - # Check for warm cache AMI (marker file baked into custom AMI) - if [[ -f /etc/vllm-ami-info ]]; then - echo "Cache: warm (custom vLLM AMI)" - cat /etc/vllm-ami-info - else - echo "Cache: cold (standard AMI)" - fi - echo "===================================" - echo "" -} - -setup_buildx_builder() { - echo "--- :buildkite: Setting up buildx builder" - if [[ -S "${BUILDKIT_SOCKET}" ]]; then - # Custom AMI with standalone buildkitd - use remote driver for warm cache - echo "✅ Found local buildkitd socket at ${BUILDKIT_SOCKET}" - echo "Using remote driver to connect to buildkitd (warm cache available)" - if docker buildx inspect baked-vllm-builder >/dev/null 2>&1; then - echo "Using existing baked-vllm-builder" - docker buildx use baked-vllm-builder - else - echo "Creating baked-vllm-builder with remote driver" - docker buildx create \ - --name baked-vllm-builder \ - --driver remote \ - --use \ - "unix://${BUILDKIT_SOCKET}" - fi - docker buildx inspect --bootstrap - elif docker buildx inspect "${BUILDER_NAME}" >/dev/null 2>&1; then - # Existing builder available - echo "Using existing builder: ${BUILDER_NAME}" - docker buildx use "${BUILDER_NAME}" - docker buildx inspect --bootstrap - else - # No local buildkitd, no existing builder - create new docker-container builder - echo "No local buildkitd found, using docker-container driver" - docker buildx create --name "${BUILDER_NAME}" --driver docker-container --use - docker buildx inspect --bootstrap - fi - - # builder info - echo "Active builder:" - docker buildx ls | grep -E '^\*|^NAME' || docker buildx ls -} - -download_ci_hcl() { - # Returns non-zero instead of exiting: the optional export path must be - # able to give up without failing a build whose image already exists. - echo "--- :arrow_down: Downloading ci.hcl" - curl -sSfL -o "${CI_HCL_PATH}" "${CI_HCL_URL}" || return $? - echo "Downloaded to ${CI_HCL_PATH}" - - if [[ ! -f "${CI_HCL_PATH}" ]]; then - echo "Error: ci.hcl not found at ${CI_HCL_PATH}" - return 1 - fi -} - -export_kernel_symbol_map_from_cache() { - # The existing-image path: everything here is best effort. Called as - # `... || echo`, so errexit is suspended inside and any failure just - # returns; the build keeps the image it already has. - local rc=0 - download_ci_hcl \ - && setup_buildx_builder \ - && resolve_parent_commit \ - && export PARENT_COMMIT \ - && BUILD_TMP_DIR="$(mktemp -d)" \ - && export_kernel_symbol_map || rc=$? - rm -rf -- "${BUILD_TMP_DIR:-}" - return "${rc}" -} - -export_kernel_symbol_map() { - # Only when the build asked for it (nightly/post-merge). The map was - # produced inside the csrc-build stage during the main bake; this second - # bake of the scratch stage is a cache hit that just writes the file out. - if [[ "${VLLM_KERNEL_SYMBOL_MAP:-0}" != "1" ]]; then - return 0 - fi - echo "--- :world_map: Exporting kernel symbol map" - local out_dir="${BUILD_TMP_DIR}/kernel-symbol-map" - mkdir -p "${out_dir}" - if docker buildx bake -f "${VLLM_BAKE_FILE_PATH}" -f "${CI_HCL_PATH}" --progress plain \ - --set "kernel-symbol-map.output=type=local,dest=${out_dir}" kernel-symbol-map \ - && [[ -s "${out_dir}/kernel_symbol_map.json.gz" ]]; then - ls -la "${out_dir}" - (cd "${out_dir}" && buildkite-agent artifact upload "kernel_symbol_map.json.gz") - else - echo "kernel symbol map export failed; continuing without it" >&2 - fi -} - -annotate_image_tags() { - .buildkite/scripts/annotate-image-build.sh \ - "${IMAGE_TAG:-}" "${IMAGE_TAG_LATEST:-}" -} - -check_and_skip_if_image_exists() { - if [[ -n "${IMAGE_TAG:-}" ]]; then - echo "--- :mag: Checking if image exists" - if docker manifest inspect "${IMAGE_TAG}" >/dev/null 2>&1; then - echo "Image already exists: ${IMAGE_TAG}" - echo "Skipping build" - if [[ "${VLLM_KERNEL_SYMBOL_MAP:-0}" == "1" ]]; then - # The image is reused, but the symbol map is an artifact of - # this build. Bake just the export target: its layers come - # from the registry cache, and the map step itself reruns - # if the cached image was built without the arg. Never - # fails the build: the image is already there. - export_kernel_symbol_map_from_cache \ - || echo "kernel symbol map: export from the cached image failed; continuing" >&2 - fi - annotate_image_tags - exit 0 - fi - echo "Image not found, proceeding with build" - fi -} - -ecr_login() { - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true - aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com || true -} - -prepare_cache_tags() { - # resolve and set: CACHE_TO, CACHE_FROM, CACHE_FROM_BASE_BRANCH, CACHE_FROM_MAIN - TEST_CACHE_ECR="936637512419.dkr.ecr.us-east-1.amazonaws.com/vllm-ci-test-cache" - MAIN_CACHE_ECR="936637512419.dkr.ecr.us-east-1.amazonaws.com/vllm-ci-postmerge-cache" - - if [[ "$BUILDKITE_PULL_REQUEST" == "false" ]]; then - if [[ "$BUILDKITE_BRANCH" == "main" ]]; then - cache="${MAIN_CACHE_ECR}:latest" - else - clean_branch=$(clean_docker_tag "$BUILDKITE_BRANCH") - cache="${TEST_CACHE_ECR}:${clean_branch}" - fi - CACHE_TO="$cache" - CACHE_FROM="$cache" - CACHE_FROM_BASE_BRANCH="$cache" - else - CACHE_TO="${TEST_CACHE_ECR}:pr-${BUILDKITE_PULL_REQUEST}" - CACHE_FROM="${TEST_CACHE_ECR}:pr-${BUILDKITE_PULL_REQUEST}" - if [[ "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" == "main" ]]; then - CACHE_FROM_BASE_BRANCH="${MAIN_CACHE_ECR}:latest" - else - clean_base=$(clean_docker_tag "$BUILDKITE_PULL_REQUEST_BASE_BRANCH") - CACHE_FROM_BASE_BRANCH="${TEST_CACHE_ECR}:${clean_base}" - fi - fi - - CACHE_FROM_MAIN="${MAIN_CACHE_ECR}:latest" - export CACHE_TO CACHE_FROM CACHE_FROM_BASE_BRANCH CACHE_FROM_MAIN -} - -resolve_parent_commit() { - if [[ -z "${PARENT_COMMIT:-}" ]]; then - PARENT_COMMIT=$(git rev-parse HEAD~1 2>/dev/null || echo "") - if [[ -n "${PARENT_COMMIT}" ]]; then - echo "Computed parent commit for cache fallback: ${PARENT_COMMIT}" - export PARENT_COMMIT - else - echo "Could not determine parent commit (may be first commit in repo)" - fi - else - echo "Using provided PARENT_COMMIT: ${PARENT_COMMIT}" - fi -} - -print_bake_config() { - echo "--- :page_facing_up: Resolved bake configuration" - # Write to a temp directory to avoid polluting the repo root (which is the - # Docker build context). Files left in the repo root get COPY'd into the - # image and can cause duplicate artifact uploads from downstream steps. - local bake_tmp - bake_tmp="$(mktemp -d)" - BAKE_CONFIG_FILE="${bake_tmp}/bake-config-build-${BUILDKITE_BUILD_NUMBER:-local}.json" - docker buildx bake "${BAKE_FILES[@]}" --print "${TARGET}" | tee "${BAKE_CONFIG_FILE}" || true - echo "Saved bake config to ${BAKE_CONFIG_FILE}" - echo "--- :arrow_down: Uploading bake config to Buildkite" - (cd "$(dirname "${BAKE_CONFIG_FILE}")" && buildkite-agent artifact upload "$(basename "${BAKE_CONFIG_FILE}")") -} - -record_buildkit_trace() ( - local metadata_file="$1" - local helper=".buildkite/scripts/ci-otel/ci_otel.py" - local trace_file="${BUILD_TMP_DIR}/buildkit-trace.json" - local build_ref record_id - - if [[ "${BUILDKITE:-}" != "true" || ! -f "${helper}" ]] || - ! command -v timeout >/dev/null 2>&1 || - ! command -v python3 >/dev/null 2>&1; then - return 0 - fi - build_ref="$(timeout 3s python3 "${helper}" build-ref "${metadata_file}" "${TARGET}" 2>/dev/null)" || { - echo "vLLM CI OTel: BuildKit metadata unavailable; trace skipped" >&2 - return 0 - } - record_id="${build_ref##*/}" - if [[ -z "${record_id}" ]]; then - return 0 - fi - - echo "--- :mag: Recording BuildKit trace" - if ! timeout 20s docker buildx history trace "${record_id}" >"${trace_file}"; then - echo "vLLM CI OTel: BuildKit history trace unavailable; upload skipped" >&2 - return 0 - fi - - export CI_INFRA_BUILDX_REF="${build_ref}" - export CI_INFRA_OTEL_SPOOL_DIR="${BUILD_TMP_DIR}/otel-spans" - timeout 5s python3 "${helper}" record-buildkit "${trace_file}" || { - echo "vLLM CI OTel: BuildKit trace conversion skipped" >&2 - return 0 - } - timeout 5s python3 "${helper}" flush || - echo "vLLM CI OTel: BuildKit trace upload skipped" >&2 -) - -################################# -# Main Script # -################################# -print_instance_info - -if [[ $# -lt 5 ]]; then - print_usage_and_exit -fi - -# input args -REGISTRY=$1 -REPO=$2 -BUILDKITE_COMMIT=$3 -BRANCH=$4 -IMAGE_TAG=$5 -IMAGE_TAG_LATEST=${6:-} # only used for main branch, optional - -# When TORCH_NIGHTLY=1, build the base CI image against PyTorch nightly so the -# entire existing pipeline runs on nightly torch (CUDA/GPU lane only). Delegate -# to the dedicated nightly build (PYTORCH_NIGHTLY=1, CUDA 13.0) and tag it at the -# normal IMAGE_TAG that every test step already pulls -- no separate image tag, -# no duplicate "vLLM Against PyTorch Nightly" pipeline section. -if [[ "${TORCH_NIGHTLY:-0}" == "1" ]]; then - echo "--- :warning: TORCH_NIGHTLY=1 -- building base image on PyTorch nightly" - SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - exec "${SCRIPT_DIR}/image_build_torch_nightly.sh" \ - "${REGISTRY}" "${REPO}" "${BUILDKITE_COMMIT}" "${BRANCH}" "${IMAGE_TAG}" -fi - -# build config -TARGET="test-ci" -VLLM_BAKE_FILE_PATH="${VLLM_BAKE_FILE_PATH:-docker/docker-bake.hcl}" -BUILDER_NAME="${BUILDER_NAME:-vllm-builder}" -CI_HCL_URL="${CI_HCL_URL:-https://raw.githubusercontent.com/vllm-project/ci-infra/main/docker/ci.hcl}" -CI_HCL_PATH="${CI_HCL_PATH:-/tmp/ci.hcl}" -ZSTD_HCL_PATH="${ZSTD_HCL_PATH:-.buildkite/image_build/zstd.hcl}" -BUILDKIT_SOCKET="/run/buildkit/buildkitd.sock" -BAKE_FILES=(-f "${VLLM_BAKE_FILE_PATH}" -f "${CI_HCL_PATH}" -f "${ZSTD_HCL_PATH}") - -prepare_cache_tags -ecr_login - -# Environment info (for docs and human readers) -# VLLM_CI_BRANCH - ci-infra branch to use (default: main) -# VLLM_BAKE_FILE_PATH - Path to vLLM's bake file (default: docker/docker-bake.hcl) -# BUILDER_NAME - Name for buildx builder (default: vllm-builder) -# -# Build configuration (exported as environment variables for bake): -export BUILDKITE_COMMIT -export PARENT_COMMIT -export IMAGE_TAG -export IMAGE_TAG_LATEST -export COMMIT="${COMMIT:-${BUILDKITE_COMMIT}}" -export CACHE_FROM -export CACHE_FROM_BASE_BRANCH -export CACHE_FROM_MAIN -export CACHE_TO - -# print args -echo "--- :mag: Arguments" -echo "REGISTRY: ${REGISTRY}" -echo "REPO: ${REPO}" -echo "BUILDKITE_COMMIT: ${BUILDKITE_COMMIT}" -echo "BRANCH: ${BRANCH}" -echo "IMAGE_TAG: ${IMAGE_TAG}" -echo "IMAGE_TAG_LATEST: ${IMAGE_TAG_LATEST}" - -# print build configuration -echo "--- :mag: Build configuration" -echo "TARGET: ${TARGET}" -echo "vLLM bake file: ${VLLM_BAKE_FILE_PATH}" -echo "BUILDER_NAME: ${BUILDER_NAME}" -echo "CI_HCL_URL: ${CI_HCL_URL}" -echo "BUILDKIT_SOCKET: ${BUILDKIT_SOCKET}" - -echo "--- :mag: Cache tags" -echo "CACHE_TO: ${CACHE_TO}" -echo "CACHE_FROM: ${CACHE_FROM}" -echo "CACHE_FROM_BASE_BRANCH: ${CACHE_FROM_BASE_BRANCH}" -echo "CACHE_FROM_MAIN: ${CACHE_FROM_MAIN}" - -check_and_skip_if_image_exists - -echo "--- :docker: Setting up Docker buildx bake" -echo "Target: ${TARGET}" -echo "vLLM bake file: ${VLLM_BAKE_FILE_PATH}" -echo "CI HCL path: ${CI_HCL_PATH}" - -if [[ ! -f "${VLLM_BAKE_FILE_PATH}" ]]; then - echo "Error: vLLM bake file not found at ${VLLM_BAKE_FILE_PATH}" - echo "Make sure you're running from the vLLM repository root" - exit 1 -fi - -download_ci_hcl || exit 1 - -setup_buildx_builder - -resolve_parent_commit -export PARENT_COMMIT - -print_bake_config - -echo "--- :docker: Building ${TARGET}" -BUILD_TMP_DIR="$(mktemp -d)" -trap 'rm -rf -- "${BUILD_TMP_DIR}"' EXIT -BUILD_METADATA_FILE="${BUILD_TMP_DIR}/build-metadata.json" -BUILD_STATUS=0 -docker --debug buildx bake "${BAKE_FILES[@]}" \ - --progress plain --metadata-file "${BUILD_METADATA_FILE}" "${TARGET}" || BUILD_STATUS=$? - -record_buildkit_trace "${BUILD_METADATA_FILE}" || true - -if [[ "${BUILD_STATUS}" -ne 0 ]]; then - exit "${BUILD_STATUS}" -fi - -echo "--- :white_check_mark: Build complete" -export_kernel_symbol_map || true - -annotate_image_tags diff --git a/.buildkite/image_build/image_build.yaml b/.buildkite/image_build/image_build.yaml deleted file mode 100644 index 35a074c98c..0000000000 --- a/.buildkite/image_build/image_build.yaml +++ /dev/null @@ -1,130 +0,0 @@ -group: Abuild -steps: - - label: ":docker: Build image" - key: image-build - depends_on: [] - timeout_in_minutes: 600 - commands: - - if [[ "$BUILDKITE_BRANCH" == "main" ]]; then .buildkite/image_build/image_build.sh $REGISTRY $REPO $BUILDKITE_COMMIT $BRANCH $IMAGE_TAG $IMAGE_TAG_LATEST; else .buildkite/image_build/image_build.sh $REGISTRY $REPO $BUILDKITE_COMMIT $BRANCH $IMAGE_TAG; fi - retry: - automatic: - - exit_status: -1 # Agent was lost - limit: 2 - - exit_status: -10 # Agent was lost - limit: 2 - - - label: ":docker: :smoking: Non-root smoke tests" - key: image-build-smoke-test - depends_on: - - image-build - commands: - # Smoke 1: the default (root) image must still be importable - # under a non-root UID via `--user 2000:0`. Validates the `vllm` passwd - # entry + group-0-writable /home/vllm + uv path cleanup from #31959. - # Uses `import vllm` rather than `vllm serve --help` because the latter - # instantiates `VllmConfig` which requires a GPU attached to the - # container. - - docker run --rm --user 2000:0 --entrypoint python3 "$IMAGE_TAG" -c "import vllm; print(vllm.__version__)" - # Smoke 2: assert the non-root enabling invariants are baked - # into the image. Runs as UID 2000:0 via a shell so we can verify - # filesystem perms + passwd/group file state + wrapper presence without - # triggering vLLM's GPU-requiring config-init path. The opt-in - # `vllm-openai-nonroot` target adds only `USER vllm`, `WORKDIR - # /home/vllm`, and an `ENTRYPOINT` override on top of these invariants; - # its build correctness is reviewed at the Dockerfile level. Wrapper - # logic is covered separately by the pre-commit hook - # `test-nonroot-entrypoint` (see .pre-commit-config.yaml). - - | - docker run --rm --user 2000:0 --entrypoint /bin/sh "$IMAGE_TAG" -ec ' - if ! getent passwd 2000 | grep -q ^vllm:; then - echo FAIL: UID 2000 != vllm - exit 1 - fi - if ! id -gn 2>/dev/null | grep -qx root; then - echo FAIL: GID 0 not root group - exit 1 - fi - touch /home/vllm/.smoke && rm /home/vllm/.smoke - touch /opt/uv/cache/.smoke && rm /opt/uv/cache/.smoke - if ! test -x /usr/local/bin/vllm-nonroot-entrypoint.sh; then - echo FAIL: wrapper missing - exit 1 - fi - if ! test -w /etc/passwd; then - echo FAIL: /etc/passwd not group-writable - exit 1 - fi - if ! test -w /etc/group; then - echo FAIL: /etc/group not group-writable - exit 1 - fi - echo non-root invariants OK - ' - retry: - automatic: - - exit_status: -1 # Agent was lost - limit: 2 - - exit_status: -10 # Agent was lost - limit: 2 - - - label: ":docker: Build CPU image" - key: image-build-cpu - depends_on: [] - commands: - - .buildkite/image_build/image_build_cpu.sh $REGISTRY $REPO $BUILDKITE_COMMIT - env: - DOCKER_BUILDKIT: "1" - retry: - automatic: - - exit_status: -1 # Agent was lost - limit: 2 - - exit_status: -10 # Agent was lost - limit: 2 - - - label: ":docker: Build HPU image" - soft_fail: true - depends_on: [] - key: image-build-hpu - commands: - - .buildkite/image_build/image_build_hpu.sh $REGISTRY $REPO $BUILDKITE_COMMIT - env: - DOCKER_BUILDKIT: "1" - retry: - automatic: - - exit_status: -1 # Agent was lost - limit: 2 - - exit_status: -10 # Agent was lost - limit: 2 - - - label: ":docker: Build CPU arm64 image" - key: cpu-arm64-image-build - depends_on: [] - optional: true - commands: - - .buildkite/image_build/image_build_cpu_arm64.sh $REGISTRY $REPO $BUILDKITE_COMMIT - env: - DOCKER_BUILDKIT: "1" - retry: - automatic: - - exit_status: -1 # Agent was lost - limit: 2 - - exit_status: -10 # Agent was lost - limit: 2 - - - label: ":docker: Build arm64 image" - key: arm64-image-build - depends_on: [] - source_file_dependencies: - - ".buildkite/image_build/image_build.yaml" - - ".buildkite/image_build/image_build_arm64.sh" - - "docker/Dockerfile" - commands: - - .buildkite/image_build/image_build_arm64.sh $REGISTRY $REPO $BUILDKITE_COMMIT - env: - DOCKER_BUILDKIT: "1" - retry: - automatic: - - exit_status: -1 # Agent was lost - limit: 2 - - exit_status: -10 # Agent was lost - limit: 2 diff --git a/.buildkite/image_build/image_build_arm64.sh b/.buildkite/image_build/image_build_arm64.sh deleted file mode 100755 index 69d7f55b9f..0000000000 --- a/.buildkite/image_build/image_build_arm64.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -set -e - -if [[ $# -lt 3 ]]; then - echo "Usage: $0 " - exit 1 -fi - -REGISTRY=$1 -REPO=$2 -BUILDKITE_COMMIT=$3 - -# When TORCH_NIGHTLY=1, build against torch nightly and tag it with the -# -torch-nightly-arm64 suffix the DGX Spark test steps pull on the nightly lane -# (ci-infra get_image(arm64=True) appends -torch-nightly before -arm64). -PYTORCH_NIGHTLY_ARGS=() -if [[ "${TORCH_NIGHTLY:-0}" == "1" ]]; then - IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-torch-nightly-arm64" - PYTORCH_NIGHTLY_ARGS=(--build-arg PYTORCH_NIGHTLY=1) -else - IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-arm64" -fi - -# authenticate with AWS ECR -aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true - -# skip build if image already exists -if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then - echo "Image found" -else - echo "Image not found, proceeding with build..." - # build for arm64 GPU targets: Grace/GH200 (sm_90), - # Blackwell/Thor (sm_100/sm_103/sm_110), and DGX Spark/GB10 - # (sm_121, family-covered by 12.0 under CUDA 13) - docker build --file docker/Dockerfile \ - --platform linux/arm64 \ - --build-arg max_jobs=16 \ - --build-arg nvcc_threads=4 \ - --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.0-b8b5f17a7d9ccfc25bbc5cf17b3fcea12964a042 \ - --build-arg torch_cuda_arch_list="9.0 10.0 11.0 12.0" \ - --build-arg USE_SCCACHE=1 \ - --build-arg buildkite_commit="$BUILDKITE_COMMIT" \ - "${PYTORCH_NIGHTLY_ARGS[@]}" \ - --tag "$IMAGE" \ - --target test \ - --progress plain . - # push - docker push "$IMAGE" -fi - -.buildkite/scripts/annotate-image-build.sh "$IMAGE" diff --git a/.buildkite/image_build/image_build_cpu.sh b/.buildkite/image_build/image_build_cpu.sh deleted file mode 100755 index 0f27538965..0000000000 --- a/.buildkite/image_build/image_build_cpu.sh +++ /dev/null @@ -1,118 +0,0 @@ -#!/bin/bash -set -e - -if [[ $# -lt 3 ]]; then - echo "Usage: $0 " - exit 1 -fi - -REGISTRY=$1 -REPO=$2 -BUILDKITE_COMMIT=$3 - -# When TORCH_NIGHTLY=1, build the CPU image against torch nightly and tag it -# with the -torch-nightly-cpu suffix the test steps pull on the nightly lane. -PYTORCH_NIGHTLY_ARGS=() -if [[ "${TORCH_NIGHTLY:-0}" == "1" ]]; then - IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-torch-nightly-cpu" - PYTORCH_NIGHTLY_ARGS=(--build-arg PYTORCH_NIGHTLY=1) -else - IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu" -fi - -# replace invalid characters in Docker image tags and truncate to 128 chars -clean_docker_tag() { - local input="$1" - echo "$input" | sed 's/[^a-zA-Z0-9._-]/_/g' | cut -c1-128 -} - -# resolve and set: CACHE_TO, CACHE_FROM, CACHE_FROM_BASE_BRANCH, CACHE_FROM_MAIN -# Reuses the same ECR cache repos as the CUDA image build, with an -# "x86_cpu" suffix on every tag so CPU and CUDA cache blobs stay isolated. -# The nightly lane appends "-nightly": it resolves a different torch, so -# sharing a cache ref with the regular lane would have the two evict each -# other from the same ref on every run. -prepare_cache_tags() { - TEST_CACHE_ECR="936637512419.dkr.ecr.us-east-1.amazonaws.com/vllm-ci-test-cache" - MAIN_CACHE_ECR="936637512419.dkr.ecr.us-east-1.amazonaws.com/vllm-ci-postmerge-cache" - CACHE_SUFFIX="x86_cpu" - if [[ "${TORCH_NIGHTLY:-0}" == "1" ]]; then - CACHE_SUFFIX="x86_cpu-nightly" - fi - - if [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" ]]; then - if [[ "${BUILDKITE_BRANCH:-}" == "main" ]]; then - cache="${MAIN_CACHE_ECR}:latest-${CACHE_SUFFIX}" - else - clean_branch=$(clean_docker_tag "${BUILDKITE_BRANCH:-unknown}") - cache="${TEST_CACHE_ECR}:${clean_branch}-${CACHE_SUFFIX}" - fi - CACHE_TO="$cache" - CACHE_FROM="$cache" - CACHE_FROM_BASE_BRANCH="$cache" - else - CACHE_TO="${TEST_CACHE_ECR}:pr-${BUILDKITE_PULL_REQUEST}-${CACHE_SUFFIX}" - CACHE_FROM="${TEST_CACHE_ECR}:pr-${BUILDKITE_PULL_REQUEST}-${CACHE_SUFFIX}" - if [[ "${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}" == "main" ]]; then - CACHE_FROM_BASE_BRANCH="${MAIN_CACHE_ECR}:latest-${CACHE_SUFFIX}" - else - clean_base=$(clean_docker_tag "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}") - CACHE_FROM_BASE_BRANCH="${TEST_CACHE_ECR}:${clean_base}-${CACHE_SUFFIX}" - fi - fi - - CACHE_FROM_MAIN="${MAIN_CACHE_ECR}:latest-${CACHE_SUFFIX}" -} - -# authenticate with AWS ECR (public, for the image; private, for the cache repos) -aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true -aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com || true - -# skip build if image already exists -if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then - echo "Image found" -else - echo "Image not found, proceeding with build..." - - prepare_cache_tags - echo "--- :mag: Cache tags" - echo "CACHE_TO: ${CACHE_TO}" - echo "CACHE_FROM: ${CACHE_FROM}" - echo "CACHE_FROM_BASE_BRANCH: ${CACHE_FROM_BASE_BRANCH}" - echo "CACHE_FROM_MAIN: ${CACHE_FROM_MAIN}" - - # dedupe cache-from refs so identical branch/main fallbacks aren't repeated - CACHE_FROM_ARGS=() - SEEN_REFS="" - for ref in "$CACHE_FROM" "$CACHE_FROM_BASE_BRANCH" "$CACHE_FROM_MAIN"; do - if [[ "$SEEN_REFS" != *"|${ref}|"* ]]; then - CACHE_FROM_ARGS+=(--cache-from "type=registry,ref=${ref}") - SEEN_REFS="${SEEN_REFS}|${ref}|" - fi - done - - echo "--- :docker: Setting up buildx" - # network=host so the isolated docker-container builder can still reach the - # EC2 instance metadata service for sccache's S3 credentials; recreate on - # every run in case a stale builder without this driver-opt already exists - # on a warm CI host. - docker buildx rm vllm-cpu-builder >/dev/null 2>&1 || true - docker buildx create --name vllm-cpu-builder --driver docker-container --driver-opt network=host --use - docker buildx inspect --bootstrap - - # build and push - docker buildx build --file docker/Dockerfile.cpu \ - --build-arg max_jobs=16 \ - --build-arg buildkite_commit="$BUILDKITE_COMMIT" \ - --build-arg VLLM_CPU_X86=true \ - --build-arg USE_SCCACHE=1 \ - "${PYTORCH_NIGHTLY_ARGS[@]}" \ - --tag "$IMAGE" \ - --target vllm-test \ - "${CACHE_FROM_ARGS[@]}" \ - --cache-to "type=registry,ref=${CACHE_TO},mode=max" \ - --push \ - --progress plain . -fi - -.buildkite/scripts/annotate-image-build.sh "$IMAGE" diff --git a/.buildkite/image_build/image_build_cpu_arm64.sh b/.buildkite/image_build/image_build_cpu_arm64.sh deleted file mode 100755 index 7bb4f10f72..0000000000 --- a/.buildkite/image_build/image_build_cpu_arm64.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -set -e - -if [[ $# -lt 3 ]]; then - echo "Usage: $0 " - exit 1 -fi - -REGISTRY=$1 -REPO=$2 -BUILDKITE_COMMIT=$3 -IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-arm64-cpu" - -# authenticate with AWS ECR -aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true - -# skip build if image already exists -if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then - echo "Image found" -else - echo "Image not found, proceeding with build..." - # build - docker build --file docker/Dockerfile.cpu \ - --build-arg max_jobs=16 \ - --build-arg buildkite_commit="$BUILDKITE_COMMIT" \ - --tag "$IMAGE" \ - --target vllm-test \ - --progress plain . - # push - docker push "$IMAGE" -fi - -.buildkite/scripts/annotate-image-build.sh "$IMAGE" diff --git a/.buildkite/image_build/image_build_hpu.sh b/.buildkite/image_build/image_build_hpu.sh deleted file mode 100755 index 35b02c7ec9..0000000000 --- a/.buildkite/image_build/image_build_hpu.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -set -e - -if [[ $# -lt 3 ]]; then - echo "Usage: $0 " - exit 1 -fi - -REGISTRY=$1 -REPO=$2 -BUILDKITE_COMMIT=$3 -IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-hpu" - -# authenticate with AWS ECR -aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true - -# skip build if image already exists -if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then - echo "Image found" -else - echo "Image not found, proceeding with build..." - # build - docker build \ - --file tests/pytorch_ci_hud_benchmark/Dockerfile.hpu \ - --build-arg max_jobs=16 \ - --build-arg buildkite_commit="$BUILDKITE_COMMIT" \ - --tag "$IMAGE" \ - --progress plain \ - https://github.com/vllm-project/vllm-gaudi.git - # push - docker push "$IMAGE" -fi - -.buildkite/scripts/annotate-image-build.sh "$IMAGE" diff --git a/.buildkite/image_build/image_build_torch_nightly.sh b/.buildkite/image_build/image_build_torch_nightly.sh deleted file mode 100755 index 4d978b5299..0000000000 --- a/.buildkite/image_build/image_build_torch_nightly.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# Build a vLLM test image with PyTorch nightly installed. -# Called by the pipeline generator's "vLLM Against PyTorch Nightly" group. - -if [[ $# -lt 5 ]]; then - echo "Usage: $0 " - exit 1 -fi - -REGISTRY=$1 -REPO=$2 -BUILDKITE_COMMIT=$3 -BRANCH=$4 -IMAGE_TAG=$5 - -# --- Arguments --- -echo "--- :mag: Arguments" -echo "REGISTRY: ${REGISTRY}" -echo "REPO: ${REPO}" -echo "BUILDKITE_COMMIT: ${BUILDKITE_COMMIT}" -echo "BRANCH: ${BRANCH}" -echo "IMAGE_TAG: ${IMAGE_TAG}" - -# --- ECR login --- -echo "--- :key: ECR login" -aws ecr-public get-login-password --region us-east-1 \ - | docker login --username AWS --password-stdin "$REGISTRY" -aws ecr get-login-password --region us-east-1 \ - | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com - -# --- Set up buildx --- -echo "--- :docker: Setting up buildx" -docker buildx create --name vllm-builder --driver docker-container --use || true -docker buildx inspect --bootstrap -docker buildx ls - -# --- Skip if image already exists --- -echo "--- :mag: Checking if image already exists" -if docker manifest inspect "$IMAGE_TAG" >/dev/null 2>&1; then - echo "Image found: $IMAGE_TAG — skipping build" - .buildkite/scripts/annotate-image-build.sh "$IMAGE_TAG" - exit 0 -fi -echo "Image not found, proceeding with build..." - -# --- CUDA 13.0 for nightly builds --- -# Nightly CI uses CUDA 13.0 while regular CI stays on CUDA 12.9 -NIGHTLY_CUDA_VERSION="13.0.2" -NIGHTLY_BUILD_BASE_IMAGE="pytorch/manylinux2_28-builder:cuda13.0-78e737ad29420ffc4800e677c51e2a852caf8359" -# Must track ARG UBUNTU_VERSION in docker/Dockerfile. The nightly lane exists to -# run the normal pipeline against nightly torch, so it has to differ from the -# regular image in torch only -- not in the OS underneath it. This was pinned to -# 22.04 while everything else moved to 24.04, and the runtime dependency list in -# tools/install_snapshot_runtime.sh is 24.04-only (libgnutls30t64 is the 64-bit -# time_t rename and does not exist on jammy), so the nightly build failed with -# "E: Unable to locate package libgnutls30t64" while regular CI stayed green on -# a cached layer. -NIGHTLY_UBUNTU_VERSION="24.04" -NIGHTLY_FINAL_BASE_IMAGE="nvidia/cuda:${NIGHTLY_CUDA_VERSION}-base-ubuntu${NIGHTLY_UBUNTU_VERSION}" - -echo "--- :docker: Building torch nightly image (CUDA ${NIGHTLY_CUDA_VERSION})" -# Provenance labels match the bake `_labels` target in docker/docker-bake.hcl; -# the initialized-snapshot E2E refuses images whose commit labels differ. -docker buildx build --file docker/Dockerfile \ - --build-arg max_jobs=16 \ - --build-arg buildkite_commit="$BUILDKITE_COMMIT" \ - --build-arg USE_SCCACHE=1 \ - --build-arg PYTORCH_NIGHTLY=1 \ - --build-arg CUDA_VERSION="${NIGHTLY_CUDA_VERSION}" \ - --build-arg BUILD_BASE_IMAGE="${NIGHTLY_BUILD_BASE_IMAGE}" \ - --build-arg FINAL_BASE_IMAGE="${NIGHTLY_FINAL_BASE_IMAGE}" \ - --build-arg torch_cuda_arch_list="8.0 8.9 9.0 10.0 12.0" \ - --label "org.opencontainers.image.revision=${BUILDKITE_COMMIT}" \ - --label "ai.vllm.build.commit=${BUILDKITE_COMMIT}" \ - --label "ai.vllm.build.pipeline=${BUILDKITE_PIPELINE_SLUG:-local}" \ - --label "ai.vllm.build.url=${BUILDKITE_BUILD_URL:-}" \ - --tag "$IMAGE_TAG" \ - --push \ - --target test \ - --progress plain . - -echo "--- :white_check_mark: Torch nightly image build complete: $IMAGE_TAG" - -.buildkite/scripts/annotate-image-build.sh "$IMAGE_TAG" diff --git a/.buildkite/image_build/image_build_xpu.sh b/.buildkite/image_build/image_build_xpu.sh deleted file mode 100755 index adc544a233..0000000000 --- a/.buildkite/image_build/image_build_xpu.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -set -e - -if [[ $# -lt 3 ]]; then - echo "Usage: $0 " - exit 1 -fi - -REGISTRY=$1 -REPO=$2 -BUILDKITE_COMMIT=$3 -IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-xpu" - -# authenticate with AWS ECR -aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true -aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com || true - -# skip build if image already exists -if docker manifest inspect "$IMAGE" &> /dev/null; then - echo "Image found" -else - echo "Image not found, proceeding with build..." - # build - docker build \ - --file docker/Dockerfile.xpu \ - --build-arg max_jobs=16 \ - --build-arg buildkite_commit="$BUILDKITE_COMMIT" \ - --tag "$IMAGE" \ - --progress plain . - # push - docker push "$IMAGE" -fi - -.buildkite/scripts/annotate-image-build.sh "$IMAGE" diff --git a/.buildkite/image_build/image_build_zen_cpu.sh b/.buildkite/image_build/image_build_zen_cpu.sh deleted file mode 100755 index 3e42e38a8d..0000000000 --- a/.buildkite/image_build/image_build_zen_cpu.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# -# Build the AMD Zen CPU image (vLLM + zentorch) by layering docker/Dockerfile.zen -# on top of a CPU base image that already has vLLM installed. -# -# To avoid recompiling vLLM from source, prefer reusing the shared CPU image that -# the image-build-cpu step already builds and publishes for this commit -# (`:-cpu`). It lives in public ECR, which allows anonymous pulls, -# so no registry credentials are required. If the pull fails (e.g. the image -# isn't published yet), fall back to building the base from docker/Dockerfile.cpu. -# -# The zen image is not pushed to a registry. -# -# See docker/Dockerfile.zen for the build workflow this mirrors. -set -euo pipefail - -if [[ $# -lt 3 ]]; then - echo "Usage: $0 " - exit 1 -fi - -REGISTRY=$1 -REPO=$2 -BUILDKITE_COMMIT=$3 - -image_repo() { - if [[ -n "${REGISTRY}" ]]; then - printf '%s/%s' "${REGISTRY}" "${REPO}" - else - printf '%s' "${REPO}" - fi -} - -IMAGE_REPO="$(image_repo)" - -# The shared CPU image published by image-build-cpu (public ECR, anonymous pull). -SHARED_CPU_IMAGE="" -if [[ -n "${REGISTRY}" ]]; then - SHARED_CPU_IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu" -fi -FALLBACK_BASE_IMAGE="zen-cpu-base:$BUILDKITE_COMMIT" -IMAGE="$IMAGE_REPO:$BUILDKITE_COMMIT-zen-cpu" - -# ZENTORCH_VERSION is optional; when unset the Dockerfile falls back to -# installing zentorch via `vllm[zen]`. -ZENTORCH_VERSION=${ZENTORCH_VERSION:-} - -# Step 1: obtain the CPU base image that Dockerfile.zen layers on. Prefer pulling -# the published `-cpu` image; fall back to building it from source if unavailable. -if [[ -n "${SHARED_CPU_IMAGE}" ]] && docker pull "$SHARED_CPU_IMAGE"; then - echo "--- :docker: Using published CPU image as base: $SHARED_CPU_IMAGE" - BASE_IMAGE="$SHARED_CPU_IMAGE" -else - echo "--- :docker: Published CPU image unavailable; building base from source" - docker build --file docker/Dockerfile.cpu \ - --platform linux/amd64 \ - --build-arg buildkite_commit="$BUILDKITE_COMMIT" \ - --build-arg VLLM_CPU_X86=true \ - --tag "$FALLBACK_BASE_IMAGE" \ - --target vllm-openai \ - --progress plain . - BASE_IMAGE="$FALLBACK_BASE_IMAGE" -fi - -# Step 2: build the zen test image on top of the CPU base. -echo "--- :docker: Building Zen test image" -docker build --file docker/Dockerfile.zen \ - --platform linux/amd64 \ - --build-arg BASE_IMAGE="$BASE_IMAGE" \ - ${ZENTORCH_VERSION:+--build-arg ZENTORCH_VERSION="$ZENTORCH_VERSION"} \ - --tag "$IMAGE" \ - --target vllm-zen-test \ - --progress plain . diff --git a/.buildkite/image_build/zstd.hcl b/.buildkite/image_build/zstd.hcl deleted file mode 100644 index 1f795c5f91..0000000000 --- a/.buildkite/image_build/zstd.hcl +++ /dev/null @@ -1,3 +0,0 @@ -target "test-ci" { - output = ["type=registry,compression=zstd,compression-level=3,force-compression=true,oci-mediatypes=true"] -} diff --git a/.buildkite/intel_jobs/basic_correctness_intel.yaml b/.buildkite/intel_jobs/basic_correctness_intel.yaml deleted file mode 100644 index 0016023dc7..0000000000 --- a/.buildkite/intel_jobs/basic_correctness_intel.yaml +++ /dev/null @@ -1,29 +0,0 @@ -group: Basic Correctness Intel -depends_on: - - image-build-xpu -steps: -- label: ":intel: (BMG) Sleep Mode" - key: xpu-sleep-mode - timeout_in_minutes: 45 - device: intel_gpu - agent_tags: - label: production - gpu: 2+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/ - - tests/basic_correctness/memory/ - - tests/basic_correctness/cpu_offload/ - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - export VLLM_WORKER_MULTIPROC_METHOD=spawn && - pytest -v -s basic_correctness/cpu_offload && - pytest -v -s basic_correctness/memory' diff --git a/.buildkite/intel_jobs/benchmarks_intel.yaml b/.buildkite/intel_jobs/benchmarks_intel.yaml deleted file mode 100644 index f6c1e104b1..0000000000 --- a/.buildkite/intel_jobs/benchmarks_intel.yaml +++ /dev/null @@ -1,26 +0,0 @@ -group: Benchmarks Intel -depends_on: - - image-build-xpu -steps: -- label: ":intel: (BMG) Benchmarks CLI" - key: benchmarks-cli-test - timeout_in_minutes: 40 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/ - - tests/benchmarks/ - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s benchmarks/' diff --git a/.buildkite/intel_jobs/distributed_intel.yaml b/.buildkite/intel_jobs/distributed_intel.yaml deleted file mode 100644 index 54800b4103..0000000000 --- a/.buildkite/intel_jobs/distributed_intel.yaml +++ /dev/null @@ -1,33 +0,0 @@ -group: Distributed Intel -depends_on: - - image-build-xpu -steps: -- label: ":intel: (BMG) Distributed Torchrun + Shutdown" - key: distributed-torchrun-shutdown-tests-2-gpus - timeout_in_minutes: 50 - device: intel_gpu - agent_tags: - label: production - gpu: 2+ - mem: 24+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/distributed/ - - "!vllm/distributed/kv_transfer/" - - vllm/engine/ - - vllm/executor/ - - vllm/worker/worker_base.py - - vllm/v1/engine/ - - vllm/v1/worker/ - - tests/distributed/ - - tests/v1/shutdown - - tests/v1/worker/test_worker_memory_snapshot.py - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && pytest -v -s v1/shutdown' diff --git a/.buildkite/intel_jobs/engine_intel.yaml b/.buildkite/intel_jobs/engine_intel.yaml deleted file mode 100644 index 5235398c25..0000000000 --- a/.buildkite/intel_jobs/engine_intel.yaml +++ /dev/null @@ -1,107 +0,0 @@ -group: Engine Intel -depends_on: - - image-build-xpu -steps: -- label: ":intel: (BMG) Engine" - key: engine - timeout_in_minutes: 40 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/compilation/ - - vllm/config/ - - vllm/engine/ - - vllm/entrypoints/logger.py - - vllm/envs.py - - vllm/logger.py - - vllm/logging_utils/ - - vllm/platforms/ - - vllm/sequence.py - - vllm/triton_utils/ - - vllm/utils/ - - tests/engine - - tests/test_sequence - - tests/test_config - - tests/test_logger - - tests/test_vllm_port - - tests/jit_monitor/test_hooks.py - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s engine/test_arg_utils.py test_sequence.py test_logger.py test_vllm_port.py jit_monitor/test_hooks.py' - -- label: ":intel: (BMG) Engine (1 GPU)" - key: engine-1-gpu - timeout_in_minutes: 30 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/v1/engine/ - - tests/v1/engine/ - - tests/test_config/ - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py --ignore v1/engine/test_core_engine_actor_manager.py && - pytest -v -s test_config.py' - -- label: ":intel: (BMG) V1 e2e (2 GPUs)" - key: v1-e2e-2-gpus - timeout_in_minutes: 30 - device: intel_gpu - agent_tags: - label: production - gpu: 2+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/compilation/ - - vllm/config/ - - vllm/distributed/ - - vllm/engine/ - - vllm/envs.py - - vllm/forward_context.py - - vllm/inputs/ - - vllm/logger.py - - vllm/logging_utils/ - - vllm/model_executor/ - - vllm/multimodal/ - - vllm/platforms/ - - vllm/sampling_params.py - - vllm/transformers_utils/ - - vllm/triton_utils/ - - vllm/utils/ - - vllm/v1/ - - tests/v1/e2e/spec_decode/ - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s - v1/e2e/spec_decode/draft_model/test_draft_model.py::test_draft_model_tensor_parallelism - v1/e2e/spec_decode/draft_model/test_draft_model.py::test_draft_model_engine_args_tensor_parallelism' diff --git a/.buildkite/intel_jobs/entrypoints_intel.yaml b/.buildkite/intel_jobs/entrypoints_intel.yaml deleted file mode 100644 index f49f051ebc..0000000000 --- a/.buildkite/intel_jobs/entrypoints_intel.yaml +++ /dev/null @@ -1,194 +0,0 @@ -group: Entrypoints Intel -depends_on: - - image-build-xpu -steps: -- label: ":intel: (BMG) Entrypoints Unit" - key: entrypoints-unit-tests-intel - timeout_in_minutes: 45 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/entrypoints - - tests/entrypoints/unit_tests - - tests/entrypoints/rl/weight_transfer - - tests/entrypoints/launchers - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s entrypoints/unit_tests && - pytest -v -s entrypoints/rl/weight_transfer' - -- label: ":intel: (BMG) Entrypoints Integration (LLM)" - key: entrypoints-integration-llm-intel - timeout_in_minutes: 70 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/ - - tests/entrypoints/llm - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'export VLLM_WORKER_MULTIPROC_METHOD=spawn && - cd tests && - pytest -v -s entrypoints/llm/test_generate.py && - pytest -v -s entrypoints/llm/offline_mode' - -- label: ":intel: (BMG) Entrypoints Integration (API Server)" - key: entrypoints-integration-api-server-intel - timeout_in_minutes: 70 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/ - - "!vllm/distributed/kv_transfer/" - - tests/entrypoints/serve - - tests/entrypoints/rl - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'export VLLM_WORKER_MULTIPROC_METHOD=spawn && - cd tests && - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc && - pytest -v -s entrypoints/rl --ignore=entrypoints/rl/weight_transfer - --ignore=entrypoints/rl/return_routed_experts/test_openai_api.py - --ignore=entrypoints/rl/weight_transfer_metrics - --deselect="tests/entrypoints/rl/state_transitions/test_pause_resume.py::TestPauseResume::test_clear_cache_preserves_output_and_controls_prefix_cache[MRV1]" && - PYTHONPATH=/workspace/vllm pytest -v -s entrypoints/serve/dev/rpc' - -- label: ":intel: (BMG) Entrypoints Integration (API Server Scale Out)" - key: entrypoints-integration-api-server-scale-out-intel - timeout_in_minutes: 40 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/ - - "!vllm/distributed/kv_transfer/" - - tests/entrypoints/scale_out - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'export VLLM_WORKER_MULTIPROC_METHOD=spawn && - cd tests && - pytest -v -s entrypoints/scale_out' - -- label: ":intel: (BMG) Entrypoints Integration (API Server Generate)" - key: entrypoints-integration-api-server-generate-intel - timeout_in_minutes: 65 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 24+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - optional: true - source_file_dependencies: - - vllm/ - - "!vllm/distributed/kv_transfer/" - - tests/tool_use - - tests/entrypoints/tool_parsers - - tests/entrypoints/generate - - tests/entrypoints/anthropic - - tests/entrypoints/cohere - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s tool_use - --deselect="tests/tool_use/mistral/test_mistral_tool_calls.py::test_tool_call_none_with_tools[ministral-3b]" - --deselect="tests/tool_use/test_chat_completions.py::test_chat_completion_with_tools[granite-3.0-8b]" - --deselect="tests/tool_use/test_chat_completions.py::test_chat_completion_with_tools[llama3.2]" && - pytest -v -s entrypoints/tool_parsers && - pytest -v -s entrypoints/generate && - pytest -v -s entrypoints/anthropic && - pytest -v -s entrypoints/cohere' - -- label: ":intel: (BMG) Entrypoints Integration (Responses API)" - key: entrypoints-integration-responses-api-intel - timeout_in_minutes: 60 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 24+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - optional: true - source_file_dependencies: - - vllm/ - - "!vllm/distributed/kv_transfer/" - - tests/entrypoints/openai/responses - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s entrypoints/openai/responses' - -- label: ":intel: (BMG) OpenAI API Correctness" - key: openai-api-correctness-intel - timeout_in_minutes: 30 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - csrc/ - - vllm/entrypoints/openai/ - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s entrypoints/openai/correctness/' diff --git a/.buildkite/intel_jobs/expert_parallelism_intel.yaml b/.buildkite/intel_jobs/expert_parallelism_intel.yaml deleted file mode 100644 index c0c95fb3a9..0000000000 --- a/.buildkite/intel_jobs/expert_parallelism_intel.yaml +++ /dev/null @@ -1,28 +0,0 @@ -group: Expert Parallelism Intel -depends_on: - - image-build-xpu -steps: -- label: ":intel: (BMG) EPLB Algorithm" - key: eplb-algorithm - timeout_in_minutes: 45 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/distributed/eplb - - tests/distributed/test_eplb_algo.py - - tests/distributed/test_eplb_utils.py - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s distributed/test_eplb_algo.py && - pytest -v -s distributed/test_eplb_utils.py' diff --git a/.buildkite/intel_jobs/kernels_intel.yaml b/.buildkite/intel_jobs/kernels_intel.yaml deleted file mode 100644 index f3a9d58f43..0000000000 --- a/.buildkite/intel_jobs/kernels_intel.yaml +++ /dev/null @@ -1,50 +0,0 @@ -group: Kernels Intel -depends_on: - - image-build-xpu -steps: -- label: ":intel: (BMG) vLLM IR Tests" - key: vllm-ir-tests - timeout_in_minutes: 30 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/ir - - vllm/kernels - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s ir && - pytest -v -s kernels/ir' - -- label: ":intel: (BMG) UVA Kernel Tests" - key: uva-kernel-tests - timeout_in_minutes: 20 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - tests/kernels/core/test_uva.py - - vllm/utils/torch_utils.py - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s kernels/core/test_uva.py' diff --git a/.buildkite/intel_jobs/lm_eval_intel.yaml b/.buildkite/intel_jobs/lm_eval_intel.yaml deleted file mode 100644 index 1919e4c850..0000000000 --- a/.buildkite/intel_jobs/lm_eval_intel.yaml +++ /dev/null @@ -1,51 +0,0 @@ -group: LM Eval Intel -depends_on: - - image-build-xpu -steps: -- label: ":intel: (BMG) GPQA Eval (GPT-OSS)" - key: xpu-gpqa-eval-gpt-oss - timeout_in_minutes: 60 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 24+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - csrc/ - - vllm/model_executor/layers/quantization - - tests/evals/gpt_oss/ - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-xpu.txt' -- label: ":intel: (BMG) LM Eval Large Models(2 GPUs)" - key: lm-eval-large-models-2-gpus - timeout_in_minutes: 45 - device: intel_gpu - agent_tags: - label: production - gpu: 2+ - mem: 24+ - no_plugin: true - optional: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/ - - .buildkite/lm-eval-harness/ - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd .buildkite/lm-eval-harness && - export VLLM_USE_DEEP_GEMM=0 && - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-xpu-fp8.txt --tp-size=2' diff --git a/.buildkite/intel_jobs/lora_intel.yaml b/.buildkite/intel_jobs/lora_intel.yaml deleted file mode 100644 index ae5e2d0165..0000000000 --- a/.buildkite/intel_jobs/lora_intel.yaml +++ /dev/null @@ -1,190 +0,0 @@ -group: LoRA Intel -depends_on: - - image-build-xpu -steps: -- label: ":intel: (BMG) LoRA Runtime + Utils" - key: lora-runtime-utils - timeout_in_minutes: 45 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/lora - - tests/lora - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - export VLLM_WORKER_MULTIPROC_METHOD=spawn && - pytest -v -s lora/test_layers.py && - pytest -v -s lora/test_lora_checkpoints.py && - pytest -v -s lora/test_lora_functions.py && - pytest -v -s lora/test_lora_huggingface.py && - pytest -v -s lora/test_lora_manager.py && - pytest -v -s lora/test_lora_utils.py && - pytest -v -s lora/test_peft_helper.py && - pytest -v -s lora/test_resolver.py && - pytest -v -s lora/test_utils.py && - pytest -v -s lora/test_worker.py' - -- label: ":intel: (BMG) LoRA Fused/MoE Kernels" - key: lora-fused-moe-kernels - timeout_in_minutes: 45 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/lora - - tests/lora - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - export VLLM_WORKER_MULTIPROC_METHOD=spawn && - pytest -v -s lora/test_fused_moe_lora_kernel.py && - pytest -v -s lora/test_moe_lora_align_sum.py --deselect="tests/lora/test_moe_lora_align_sum.py::test_moe_lora_align_block_size_mixed_base_and_lora[1]"' - -- label: ":intel: (BMG) LoRA Punica Kernels" - key: lora-punica-kernels - timeout_in_minutes: 45 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/lora - - tests/lora - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - export VLLM_WORKER_MULTIPROC_METHOD=spawn && - set -o pipefail && - pytest -v -s lora/test_punica_ops.py::test_kernels && - pytest -v -s lora/test_punica_ops.py::test_kernels_hidden_size && - pytest -v -s lora/test_punica_ops.py::test_add_lora_fused_moe_early_exit' - -- label: ":intel: (BMG) LoRA Punica FP8/XPU Ops" - key: lora-punica-fp8-xpu-ops - timeout_in_minutes: 60 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/lora - - tests/lora - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - export VLLM_WORKER_MULTIPROC_METHOD=spawn && - pytest -v -s lora/test_punica_ops_fp8.py && - pytest -v -s lora/test_punica_xpu_ops.py' - -- label: ":intel: (BMG) LoRA Models" - key: lora-models - timeout_in_minutes: 45 - device: intel_gpu - agent_tags: - label: production - gpu: 2+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/lora - - tests/lora - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - export VLLM_WORKER_MULTIPROC_METHOD=spawn && - pytest -v -s lora/test_quant_model.py --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model0]" --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model1]" --deselect="tests/lora/test_quant_model.py::test_quant_model_tp_equality[model0]" && - pytest -v -s lora/test_transformers_model.py && - pytest -v -s lora/test_chatglm3_tp.py && - pytest -v -s lora/test_llama_tp.py::test_llama_lora' - -- label: ":intel: (BMG) LoRA Multimodal" - key: lora-multimodal - timeout_in_minutes: 45 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/lora - - tests/lora - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - export VLLM_WORKER_MULTIPROC_METHOD=spawn && - pytest -v -s lora/test_whisper.py' - -- label: ":intel: (BMG) LoRA Add + Multimodal" - key: lora-add-multimodal - timeout_in_minutes: 45 - device: intel_gpu - agent_tags: - label: production - gpu: 2+ - mem: 24+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/lora - - tests/lora - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - export VLLM_WORKER_MULTIPROC_METHOD=spawn && - pytest -v -s lora/test_add_lora.py && - pytest -v -s lora/test_default_mm_loras.py && - pytest -s -v lora/test_minicpmv_tp.py' diff --git a/.buildkite/intel_jobs/misc_intel.yaml b/.buildkite/intel_jobs/misc_intel.yaml deleted file mode 100644 index c1706d70f6..0000000000 --- a/.buildkite/intel_jobs/misc_intel.yaml +++ /dev/null @@ -1,391 +0,0 @@ -group: Miscellaneous Intel -depends_on: - - image-build-xpu -steps: -- label: ":intel: (BMG) Batch Invariance" - key: xpu-batch-invariance - timeout_in_minutes: 60 - num_devices: 2 - device: intel_gpu - agent_tags: - label: production - gpu: 2+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/ - - tests/v1/determinism/test_xpu_batch_invariant_ut.py - - .buildkite/scripts/hardware_ci/run-xpu-batch-invariance.sh - - .buildkite/intel_jobs/misc_intel.yaml - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'export VLLM_BATCH_INVARIANT=1 && - cd tests && - pytest -v -s v1/determinism/test_xpu_batch_invariant_ut.py && - bash ../.buildkite/scripts/hardware_ci/run-xpu-batch-invariance.sh' - -- label: ":intel: (BMG) V1 Core + KV + Metrics" - key: v1-core-kv-metrics-intel - timeout_in_minutes: 45 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/ - - tests/v1/core - - tests/v1/executor - - tests/v1/kv_offload - - tests/v1/worker - - tests/v1/kv_connector/unit - - tests/v1/ec_connector/unit - - tests/v1/metrics - - tests/entrypoints/openai/correctness/test_lmeval.py - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'pip install -r requirements/kv_connectors.txt && - export VLLM_WORKER_MULTIPROC_METHOD=spawn && - cd tests && - pytest -v -s v1/executor && - VLLM_BATCH_INVARIANT=1 pytest -v -s -m "not cpu_test" v1/ec_connector/unit - --ignore=v1/ec_connector/unit/test_ec_cpu_connector.py' - -- label: ":intel: (BMG) EC CPU Connector (XPU)" - key: ec-cpu-connector-xpu - timeout_in_minutes: 45 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 24+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/ - - tests/v1/core - - tests/v1/executor - - tests/v1/kv_offload - - tests/v1/worker - - tests/v1/kv_connector/unit - - tests/v1/ec_connector/unit - - tests/v1/metrics - - tests/entrypoints/openai/correctness/test_lmeval.py - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'pip install -r requirements/kv_connectors.txt && - export VLLM_WORKER_MULTIPROC_METHOD=spawn && - cd tests && - pytest -v -s v1/ec_connector/unit/test_ec_cpu_connector.py' - -- label: ":intel: (BMG) V1 Sample + Logits" - key: v1-sample-logits-intel - timeout_in_minutes: 90 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/config/ - - vllm/distributed/ - - vllm/engine/ - - vllm/inputs/ - - vllm/logger.py - - vllm/model_executor/ - - vllm/platforms/ - - vllm/sampling_params.py - - vllm/transformers_utils/ - - vllm/utils/ - - vllm/v1/ - - tests/v1/sample - - tests/v1/logits_processors - - tests/v1/test_oracle.py - - tests/v1/test_request.py - - tests/v1/test_outputs.py - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'export VLLM_WORKER_MULTIPROC_METHOD=spawn && - cd tests && - pytest -v -s v1/logits_processors --ignore=v1/logits_processors/test_custom_online.py --ignore=v1/logits_processors/test_custom_offline.py && - pytest -v -s v1/test_oracle.py v1/test_request.py v1/test_outputs.py && - pytest -v -s v1/sample' - retry: - automatic: - - exit_status: 1 # Test command failed - limit: 1 - parallelism: 4 - -- label: ":intel: (BMG) Basic Models Tests (Initialization)" - key: basic-models-tests-initialization - timeout_in_minutes: 60 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/ - - tests/models/test_initialization.py - - tests/models/registry.py - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'export VLLM_XPU_FUSED_MOE_USE_REF=1 && - cd tests && - pytest -v -s models/test_initialization.py::test_can_initialize_large_subset[Eagle3MiniMaxM2ForCausalLM]' - -- label: ":intel: (BMG) CPU Offload" - key: xpu-cpu-offload - timeout_in_minutes: 60 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/ - - vllm/v1/kv_offload/ - - vllm/v1/kv_connector/ - - tests/v1/kv_offload/ - - tests/v1/kv_connector/unit/test_offloading_connector.py - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'export VLLM_WORKER_MULTIPROC_METHOD=spawn && - cd tests && - pytest -v -s v1/kv_offload && - pytest -v -s v1/kv_connector/unit/test_offloading_connector.py' - -- label: ":intel: (BMG) NixlConnector PD accuracy (4 GPUs)" - key: nixlconnector-pd-accuracy-4-gpus - timeout_in_minutes: 60 - num_devices: 4 - device: intel_gpu - agent_tags: - label: production - gpu: 4+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/distributed/kv_transfer/kv_connector/ - - vllm/v1/worker/kv_connector_model_runner_mixin.py - - tests/v1/kv_connector/nixl_integration/ - - vllm/platforms/xpu.py - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - bash v1/kv_connector/nixl_integration/run_xpu_disagg_accuracy_test.sh && - PREFILLER_TP_SIZE=2 DECODER_TP_SIZE=1 bash v1/kv_connector/nixl_integration/run_xpu_disagg_accuracy_test.sh && - PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2 bash v1/kv_connector/nixl_integration/run_xpu_disagg_accuracy_test.sh && - PREFILLER_TP_SIZE=2 DECODER_TP_SIZE=2 bash v1/kv_connector/nixl_integration/run_xpu_disagg_accuracy_test.sh' - -- label: ":intel: (BMG) Regression" - key: regression - timeout_in_minutes: 50 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/config/ - - vllm/distributed/ - - vllm/engine/ - - vllm/inputs/ - - vllm/model_executor/ - - vllm/multimodal/ - - vllm/platforms/ - - vllm/sampling_params.py - - vllm/transformers_utils/ - - vllm/utils/ - - vllm/v1/ - - tests/test_regression - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s test_regression.py' - -- label: ":intel: (BMG) Metrics, Tracing (2 GPUs)" - key: metrics-tracing-2-gpus - timeout_in_minutes: 45 - num_devices: 2 - device: intel_gpu - agent_tags: - label: production - gpu: 2+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/config/ - - vllm/distributed/ - - vllm/engine/ - - vllm/inputs/ - - vllm/model_executor/ - - vllm/multimodal/ - - vllm/platforms/ - - vllm/sampling_params.py - - vllm/tracing/ - - vllm/transformers_utils/ - - vllm/utils/ - - vllm/v1/ - - tests/v1/tracing - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s v1/tracing' - -- label: ":intel: (BMG) Async Engine, Inputs, Utils, Worker" - key: async-engine-inputs-utils-worker - timeout_in_minutes: 55 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/assets/ - - vllm/config/ - - vllm/distributed/ - - vllm/engine/ - - vllm/inputs/ - - vllm/model_executor/ - - vllm/multimodal/ - - vllm/platforms/ - - vllm/sampling_params.py - - vllm/tokenizers/ - - vllm/transformers_utils/ - - vllm/utils/ - - vllm/v1/ - - tests/detokenizer - - tests/multimodal - - tests/utils_ - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s detokenizer --ignore=detokenizer/test_stop_strings.py && - pytest -v -s -m "not cpu_test" ./multimodal && - pytest -v -s utils_ --ignore=utils_/test_mem_utils.py' - -- label: ":intel: (BMG) Detokenizer Stop Strings (XPU)" - key: detokenizer-stop-strings-xpu - timeout_in_minutes: 45 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 24+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/assets/ - - vllm/config/ - - vllm/distributed/ - - vllm/engine/ - - vllm/inputs/ - - vllm/model_executor/ - - vllm/multimodal/ - - vllm/platforms/ - - vllm/sampling_params.py - - vllm/tokenizers/ - - vllm/transformers_utils/ - - vllm/utils/ - - vllm/v1/ - - tests/detokenizer - - tests/multimodal - - tests/utils_ - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s detokenizer/test_stop_strings.py' - -- label: ":intel: (BMG) Fusion Unit Tests" - key: fusion-unit-tests - timeout_in_minutes: 30 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/compilation/ - - tests/compile/passes/test_qk_norm_rope_fusion.py - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s compile/passes/test_qk_norm_rope_fusion.py' diff --git a/.buildkite/intel_jobs/model_executor_intel.yaml b/.buildkite/intel_jobs/model_executor_intel.yaml deleted file mode 100644 index 3c9b80cbc2..0000000000 --- a/.buildkite/intel_jobs/model_executor_intel.yaml +++ /dev/null @@ -1,59 +0,0 @@ -group: Model Executor Intel -depends_on: - - image-build-xpu -steps: -- label: ":intel: (BMG) Model Executor" - key: model-executor-intel - timeout_in_minutes: 45 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/engine/arg_utils.py - - vllm/config/model.py - - vllm/model_executor - - tests/model_executor - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'apt-get update && apt-get install -y curl libsodium23 && - export VLLM_WORKER_MULTIPROC_METHOD=spawn && - export PYTHONFAULTHANDLER=1 && - cd tests && - pytest -v -s model_executor -m "not slow_test" --ignore="model_executor/layers/test_rocm_unquantized_gemm.py" --ignore="model_executor/model_loader/tensorizer_loader/test_tensorizer.py" --ignore="model_executor/model_loader/test_reload.py" --deselect="tests/model_executor/test_enabled_custom_ops.py::test_topk_gating_enabled_by_cdna"' - -- label: ":intel: (BMG) Tensorizer Model Loader Tests (XPU)" - key: tensorizer-model-loader-tests-xpu - timeout_in_minutes: 45 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 24+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/engine/arg_utils.py - - vllm/config/model.py - - vllm/model_executor - - tests/model_executor - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'apt-get update && apt-get install -y curl libsodium23 && - export VLLM_WORKER_MULTIPROC_METHOD=spawn && - export PYTHONFAULTHANDLER=1 && - cd tests && - pytest -v -s model_executor/model_loader/tensorizer_loader/test_tensorizer.py -m "not slow_test"' diff --git a/.buildkite/intel_jobs/models_distributed_intel.yaml b/.buildkite/intel_jobs/models_distributed_intel.yaml deleted file mode 100644 index 5fd723f8c5..0000000000 --- a/.buildkite/intel_jobs/models_distributed_intel.yaml +++ /dev/null @@ -1,27 +0,0 @@ -group: Models - Distributed Intel -depends_on: - - image-build-xpu -steps: -- label: ":intel: (BMG) Distributed Model Tests" - key: distributed-model-tests-2-gpus - timeout_in_minutes: 65 - device: intel_gpu - agent_tags: - label: production - gpu: 2+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/model_executor/model_loader/sharded_state_loader.py - - vllm/model_executor/models/ - - tests/model_executor/model_loader/test_sharded_state_loader.py - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m "not slow_test"' diff --git a/.buildkite/intel_jobs/models_multimodal_intel.yaml b/.buildkite/intel_jobs/models_multimodal_intel.yaml deleted file mode 100644 index 9a9790d5ad..0000000000 --- a/.buildkite/intel_jobs/models_multimodal_intel.yaml +++ /dev/null @@ -1,153 +0,0 @@ -group: Models - Multimodal Intel -depends_on: - - image-build-xpu -steps: -- label: ":intel: (BMG) Multi-Modal Models (Standard) 1: qwen2" - key: multi-modal-models-standard-1-qwen2 - timeout_in_minutes: 70 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/ - - tests/models/multimodal - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2" && - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model' - -- label: ":intel: (BMG) Multi-Modal Models (Standard) 2: qwen3 + gemma" - key: multi-modal-models-standard-2-qwen3-gemma - timeout_in_minutes: 90 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/ - - tests/models/multimodal - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma" && - pytest -v -s models/multimodal/generation/test_mm_prefix_lm.py -m core_model && - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model' - -- label: ":intel: (BMG) Multi-Modal Models (Standard) 3: llava + qwen2_vl" - key: multi-modal-models-standard-3-llava-qwen2-vl - timeout_in_minutes: 65 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 24+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/ - - tests/models/multimodal - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma" && - pytest -v -s models/multimodal/generation/test_gemma_ragged_audio.py -m core_model - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model' - -- label: ":intel: (BMG) Multi-Modal Models (Standard) 4: other + whisper" - key: multi-modal-models-standard-4-other-whisper - timeout_in_minutes: 45 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/ - - tests/models/multimodal - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_gemma_ragged_audio.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing' - -- label: ":intel: (BMG) Multi-Modal Processor" # 44min - key: multi-modal-processor - timeout_in_minutes: 60 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 24+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/ - - tests/models/multimodal - - tests/models/registry.py - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s models/multimodal/processing/test_tensor_schema.py - --deselect="tests/models/multimodal/processing/test_tensor_schema.py::test_model_tensor_schema[deepseek-ai/DeepSeek-V4.1-Flash]" - --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB' - parallelism: 4 - -- label: ":intel: (BMG) Multi-Modal Models (Extended Generation 1)" - key: multi-modal-models-extended-generation-1 - timeout_in_minutes: 60 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - optional: true - source_file_dependencies: - - vllm/ - - "!vllm/distributed/kv_transfer/" - - tests/models/multimodal/generation - - tests/models/multimodal/test_mapping.py - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s models/multimodal/test_mapping.py' diff --git a/.buildkite/intel_jobs/quantization_intel.yaml b/.buildkite/intel_jobs/quantization_intel.yaml deleted file mode 100644 index 035ad11bee..0000000000 --- a/.buildkite/intel_jobs/quantization_intel.yaml +++ /dev/null @@ -1,31 +0,0 @@ -group: Quantization Intel -depends_on: - - image-build-xpu -steps: -- label: ":intel: (BMG) Quantization" - key: quantization - timeout_in_minutes: 30 - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - no_plugin: true - working_dir: "." - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - source_file_dependencies: - - csrc/ - - vllm/model_executor/layers/quantization - - tests/quantization - commands: - # - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s tests/quantization/test_per_token_kv_cache.py --deselect="tests/quantization/test_per_token_kv_cache.py::test_triton_unified_attention_per_token_head_scale[int4-16-128-num_heads0-seq_lens1]" - && VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s tests/quantization/test_auto_awq.py - && VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s tests/quantization/test_fp8_per_channel.py - && VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s tests/models/quantization/test_fp8_per_channel.py --deselect="tests/models/quantization/test_fp8_per_channel.py::test_fp8_per_channel_logprobs[moe]"' - diff --git a/.buildkite/intel_jobs/samplers_intel.yaml b/.buildkite/intel_jobs/samplers_intel.yaml deleted file mode 100644 index ed55c49da8..0000000000 --- a/.buildkite/intel_jobs/samplers_intel.yaml +++ /dev/null @@ -1,56 +0,0 @@ -group: Samplers Intel -depends_on: - - image-build-xpu -steps: -- label: ":intel: (BMG) Samplers Test (XPU Kernel)" - key: samplers-test-xpu-kernel-intel - timeout_in_minutes: 40 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/model_executor/layers - - vllm/sampling_metadata.py - - tests/samplers - - tests/conftest.py - - vllm/entrypoints/generate/beam_search - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - VLLM_XPU_USE_SAMPLER_KERNEL=1 pytest -v -s samplers - --ignore=samplers/test_beam_search.py' - -- label: ":intel: (BMG) Samplers Beam Search Test (XPU Kernel)" - key: samplers-beam-search-test-xpu-kernel-intel - timeout_in_minutes: 40 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 24+ - no_plugin: true - working_dir: "." - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/model_executor/layers - - vllm/sampling_metadata.py - - tests/samplers - - tests/conftest.py - - vllm/entrypoints/generate/beam_search - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - VLLM_XPU_USE_SAMPLER_KERNEL=1 pytest -v -s samplers/test_beam_search.py' diff --git a/.buildkite/intel_jobs/test-intel.yaml b/.buildkite/intel_jobs/test-intel.yaml deleted file mode 100644 index 74d6eee490..0000000000 --- a/.buildkite/intel_jobs/test-intel.yaml +++ /dev/null @@ -1,240 +0,0 @@ -group: Intel -steps: - - label: ":docker: Build XPU image" - soft_fail: true - depends_on: [] - key: image-build-xpu - commands: - - bash -lc '.buildkite/image_build/image_build_xpu.sh "public.ecr.aws/q9t5s3a7" "vllm-ci-test-repo" "$BUILDKITE_COMMIT"' - env: - DOCKER_BUILDKIT: "1" - retry: - automatic: - - exit_status: -1 # Agent was lost - limit: 2 - - exit_status: -10 # Agent was lost - limit: 2 - - label: ":intel: (BMG) Example Test" - key: xpu-example-test - depends_on: - - image-build-xpu - timeout_in_minutes: 70 - device: intel_gpu - agent_tags: - label: production - gpu: 2+ - mem: 16+ - no_plugin: true - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - source_file_dependencies: - - vllm/ - - .buildkite/intel_jobs/test-intel.yaml - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager && - python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 -O3 -cc.cudagraph_mode=NONE && - python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp && - python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN && - python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8_per_tensor && - python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8 && - python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096 && - python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192 && - python3 examples/basic/offline_inference/generate.py --model TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ --block-size 64 --enforce-eager && - python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 && - python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel && - python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --max-model-len 8192 && - python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-LLMC -tp 2 --max-model-len 8192 - ' - - label: ":intel: (BMG) W8A8 FP8 Linear Examples" - key: xpu-w8a8-fp8-linear-examples - depends_on: - - image-build-xpu - timeout_in_minutes: 60 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/ - - .buildkite/intel_jobs/test-intel.yaml - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096 && - python3 examples/basic/offline_inference/generate.py --linear-backend torch --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096 && - python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic --enforce-eager --max-model-len 4096 && - python3 examples/basic/offline_inference/generate.py --linear-backend torch --model Qwen/Qwen3-4B-Instruct-2507-FP8 --enforce-eager --max-model-len 4096 && - python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8_per_tensor --enforce-eager --max-model-len 4096 && - python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8_per_tensor --enforce-eager --max-model-len 4096 - ' - - label: ":intel: (BMG) V1 test" - key: xpu-v1-test - depends_on: - - image-build-xpu - timeout_in_minutes: 70 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - source_file_dependencies: - - vllm/ - - .buildkite/intel_jobs/test-intel.yaml - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && pip install dpctl && - pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py && - pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py --ignore=v1/engine/test_core_engine_actor_manager.py && - pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp" && - pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py && - pytest -v -s v1/structured_output && - pytest -v -s v1/test_serial_utils.py && - pytest -v -s v1/e2e/general/test_correctness_sliding_window.py && - pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py --deselect=tests/v1/spec_decode/test_mtp.py::test_glm_mtp_defers_lm_head && - pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --deselect=tests/v1/kv_connector/unit/test_nixl_connector.py::test_abort_timeout_on_prefiller[ray]' - parallelism: 2 - - label: ":intel: (BMG) Qwen3-30B + V1 KV Connector" - key: xpu-qwen3-30b-v1-kv-connector - depends_on: - - image-build-xpu - timeout_in_minutes: 90 - device: intel_gpu - agent_tags: - label: production - gpu: 2+ - mem: 24+ - no_plugin: true - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - source_file_dependencies: - - vllm/ - - .buildkite/intel_jobs/test-intel.yaml - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pip install dpctl && - pytest -v -s v1/kv_connector/unit/test_multi_connector.py && - pytest -v -s v1/kv_connector/unit/test_example_connector.py && - cd .. && - VLLM_XPU_FUSED_MOE_USE_REF=1 python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 --enforce-eager -tp 2 --max-model-len 8192' - - label: ":intel: (BMG) Server test" - key: xpu-server-test - depends_on: - - image-build-xpu - timeout_in_minutes: 45 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - source_file_dependencies: - - vllm/ - - .buildkite/intel_jobs/test-intel.yaml - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py && - pytest -v -s benchmarks/test_serve_cli.py' - - label: ":intel: (BMG) Quantization test" - key: xpu-quantization-test - depends_on: - - image-build-xpu - timeout_in_minutes: 30 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - source_file_dependencies: - - vllm/ - - .buildkite/intel_jobs/test-intel.yaml - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s quantization/test_auto_round.py && - pytest -v -s quantization/test_online.py --deselect=tests/quantization/test_online.py::test_online_quantization_loads_real_weights --deselect="tests/quantization/test_online.py::test_online_quantization[False-requantization_mxfp8_ptcp_fp8]"' - - label: ":intel: (BMG) Compressed tensors FP8 test" - key: xpu-compressed-tensors-fp8-test - depends_on: - - image-build-xpu - timeout_in_minutes: 60 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 16+ - no_plugin: true - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/ - - tests/quantization/test_compressed_tensors.py - - .buildkite/intel_jobs/test-intel.yaml - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s quantization/test_compressed_tensors.py::test_compressed_tensors_fp8 && - pytest -v -s quantization/test_compressed_tensors.py::test_compressed_tensors_fp8_block_enabled' - - label: ":intel: (BMG) Graph" - key: xpu-graph - depends_on: - - image-build-xpu - timeout_in_minutes: 60 - device: intel_gpu - agent_tags: - label: production - gpu: 1+ - mem: 32+ - no_plugin: true - env: - REGISTRY: "public.ecr.aws/q9t5s3a7" - REPO: "vllm-ci-test-repo" - VLLM_TEST_DEVICE: "xpu" - source_file_dependencies: - - vllm/compilation/ - - vllm/config/compilation.py - - vllm/platforms/xpu.py - - vllm/v1/cudagraph_dispatcher.py - - vllm/v1/worker/encoder_cudagraph.py - - vllm/v1/worker/encoder_cudagraph_defs.py - - vllm/v1/worker/xpu_model_runner.py - - vllm/v1/worker/xpu_worker.py - - tests/v1/cudagraph - - .buildkite/intel_jobs/test-intel.yaml - commands: - - >- - bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && - pytest -v -s v1/cudagraph/test_cudagraph_dispatch.py && - pytest -v -s v1/cudagraph/test_cudagraph_mode.py && - pytest -v -s v1/cudagraph/test_breakable_cudagraph.py && - pytest -v -s v1/cudagraph/test_encoder_cudagraph.py' diff --git a/.buildkite/lm-eval-harness/configs/DeepSeek-V2-Lite-Chat.yaml b/.buildkite/lm-eval-harness/configs/DeepSeek-V2-Lite-Chat.yaml deleted file mode 100644 index d392a5f640..0000000000 --- a/.buildkite/lm-eval-harness/configs/DeepSeek-V2-Lite-Chat.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# For vllm script, with -t option (tensor parallel size). -# bash ./run-lm-eval-gsm-vllm-baseline.sh -m deepseek-ai/DeepSeek-V2-Lite-Chat -b "auto" -l 1000 -f 5 -t 2 -model_name: "deepseek-ai/DeepSeek-V2-Lite-Chat" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.671 - - name: "exact_match,flexible-extract" - value: 0.664 -limit: 1000 -num_fewshot: 5 -trust_remote_code: True \ No newline at end of file diff --git a/.buildkite/lm-eval-harness/configs/DeepSeek-V4-Flash-MXFP4.yaml b/.buildkite/lm-eval-harness/configs/DeepSeek-V4-Flash-MXFP4.yaml deleted file mode 100644 index 2e6a74f8cc..0000000000 --- a/.buildkite/lm-eval-harness/configs/DeepSeek-V4-Flash-MXFP4.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# DeepSeek-V4-Flash MXFP4 (AMD Quark) accuracy test. -# Runs on ROCm (gfx950) with the AITER MoE + fused shared-experts path. -# Reference GSM8K (8-shot, lm-eval): strict 0.9522 / flexible 0.9515 measured -# on gfx950 TP=4 (AITER_MXFP4_BF16 backend); default rtol=0.08. -model_name: "amd/DeepSeek-V4-Flash-MXFP4" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.95 - - name: "exact_match,flexible-extract" - value: 0.95 -limit: 1319 -num_fewshot: 8 -max_model_len: 8192 -kv_cache_dtype: fp8 -moe_backend: aiter -enforce_eager: false -trust_remote_code: true -tokenizer_mode: deepseek_v4 -required_gpu_arch: -- gfx950 -env_vars: - VLLM_ROCM_USE_AITER: "1" - VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: "1" diff --git a/.buildkite/lm-eval-harness/configs/Meta-Llama-3-70B-Instruct-FBGEMM-nonuniform.yaml b/.buildkite/lm-eval-harness/configs/Meta-Llama-3-70B-Instruct-FBGEMM-nonuniform.yaml deleted file mode 100644 index 4b7776b20d..0000000000 --- a/.buildkite/lm-eval-harness/configs/Meta-Llama-3-70B-Instruct-FBGEMM-nonuniform.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# For hf script, without -t option (tensor parallel size). -# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh -m nm-testing/Meta-Llama-3-70B-Instruct-FBGEMM-nonuniform -b auto -l 1000 -f 5 -model_name: "nm-testing/Meta-Llama-3-70B-Instruct-FBGEMM-nonuniform" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.905 - - name: "exact_match,flexible-extract" - value: 0.905 -limit: 1000 -num_fewshot: 5 diff --git a/.buildkite/lm-eval-harness/configs/Meta-Llama-3-70B-Instruct.yaml b/.buildkite/lm-eval-harness/configs/Meta-Llama-3-70B-Instruct.yaml deleted file mode 100644 index 05b6617519..0000000000 --- a/.buildkite/lm-eval-harness/configs/Meta-Llama-3-70B-Instruct.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# For hf script, without -t option (tensor parallel size). -# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh -m meta-llama/Meta-Llama-3-70B-Instruct -b 32 -l 250 -f 5 -model_name: "meta-llama/Meta-Llama-3-70B-Instruct" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.892 - - name: "exact_match,flexible-extract" - value: 0.892 -limit: 250 -num_fewshot: 5 diff --git a/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-Channelwise-compressed-tensors.yaml b/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-Channelwise-compressed-tensors.yaml deleted file mode 100644 index 12a87e5290..0000000000 --- a/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-Channelwise-compressed-tensors.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# For vllm script, with -t option (tensor parallel size). -# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-Instruct-W8A8-FP8-Channelwise-compressed-tensors -b auto -l 1000 -f 5 -t 1 -model_name: "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-FP8-Channelwise-compressed-tensors" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.752 - - name: "exact_match,flexible-extract" - value: 0.754 -limit: 1000 -num_fewshot: 5 diff --git a/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-FBGEMM-nonuniform.yaml b/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-FBGEMM-nonuniform.yaml deleted file mode 100644 index 7c7a1ca6ed..0000000000 --- a/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-FBGEMM-nonuniform.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# For vllm script, with -t option (tensor parallel size). -# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-Instruct-FBGEMM-nonuniform -b auto -l 1000 -f 5 -t 1 -model_name: "nm-testing/Meta-Llama-3-8B-Instruct-FBGEMM-nonuniform" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.753 - - name: "exact_match,flexible-extract" - value: 0.753 -limit: 1000 -num_fewshot: 5 diff --git a/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-FP8-compressed-tensors.yaml b/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-FP8-compressed-tensors.yaml deleted file mode 100644 index 1d45c37704..0000000000 --- a/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-FP8-compressed-tensors.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# For vllm script, with -t option (tensor parallel size). -# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test -b 32 -l 1000 -f 5 -t 1 -model_name: "nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.755 - - name: "exact_match,flexible-extract" - value: 0.755 -limit: 1000 -num_fewshot: 5 diff --git a/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-FP8.yaml b/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-FP8.yaml deleted file mode 100644 index 29a145252e..0000000000 --- a/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-FP8.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# For vllm script, with -t option (tensor parallel size). -# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m neuralmagic/Meta-Llama-3-8B-Instruct-FP8 -b 32 -l 250 -f 5 -t 1 -model_name: "neuralmagic/Meta-Llama-3-8B-Instruct-FP8" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.753 - - name: "exact_match,flexible-extract" - value: 0.753 -limit: 1000 -num_fewshot: 5 diff --git a/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-INT8-compressed-tensors-asym.yaml b/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-INT8-compressed-tensors-asym.yaml deleted file mode 100644 index 3a5f120b3e..0000000000 --- a/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-INT8-compressed-tensors-asym.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# For vllm script, with -t option (tensor parallel size). -# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-Instruct-W8-Channel-A8-Dynamic-Asym-Per-Token-Test -b "auto" -l 250 -f 5 -t 1 -model_name: "nm-testing/Meta-Llama-3-8B-Instruct-W8-Channel-A8-Dynamic-Asym-Per-Token-Test" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.764 - - name: "exact_match,flexible-extract" - value: 0.764 -limit: 250 -num_fewshot: 5 diff --git a/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-INT8-compressed-tensors.yaml b/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-INT8-compressed-tensors.yaml deleted file mode 100644 index 5ff57bae49..0000000000 --- a/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-INT8-compressed-tensors.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# For vllm script, with -t option (tensor parallel size). -# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-Instruct-W8-Channel-A8-Dynamic-Per-Token-Test -b "auto" -l 250 -f 5 -t 1 -model_name: "nm-testing/Meta-Llama-3-8B-Instruct-W8-Channel-A8-Dynamic-Per-Token-Test" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.728 - - name: "exact_match,flexible-extract" - value: 0.728 -limit: 250 -num_fewshot: 5 diff --git a/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-nonuniform-compressed-tensors.yaml b/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-nonuniform-compressed-tensors.yaml deleted file mode 100644 index 07fb130464..0000000000 --- a/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-nonuniform-compressed-tensors.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# For vllm script, with -t option (tensor parallel size). -# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-Instruct-nonuniform-test -b auto -l 1000 -f 5 -t 1 -model_name: "nm-testing/Meta-Llama-3-8B-Instruct-nonuniform-test" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.758 - - name: "exact_match,flexible-extract" - value: 0.759 -limit: 1000 -num_fewshot: 5 diff --git a/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct.yaml b/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct.yaml deleted file mode 100644 index c27886525b..0000000000 --- a/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# For hf script, without -t option (tensor parallel size). -# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh -m meta-llama/Meta-Llama-3-8B-Instruct -b 32 -l 250 -f 5 -model_name: "meta-llama/Meta-Llama-3-8B-Instruct" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.756 - - name: "exact_match,flexible-extract" - value: 0.752 -limit: 250 -num_fewshot: 5 diff --git a/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-QQQ.yaml b/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-QQQ.yaml deleted file mode 100644 index 56ec933c9c..0000000000 --- a/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-QQQ.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# For vllm script, with -t option (tensor parallel size). -# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m HandH1998/QQQ-Llama-3-8b-g128 -b 32 -l 1000 -f 5 -t 1 -model_name: "HandH1998/QQQ-Llama-3-8b-g128" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.419 - - name: "exact_match,flexible-extract" - value: 0.416 -limit: 1000 -num_fewshot: 5 diff --git a/.buildkite/lm-eval-harness/configs/Meta-Llama-3.2-1B-Instruct-FP8-compressed-tensors.yaml b/.buildkite/lm-eval-harness/configs/Meta-Llama-3.2-1B-Instruct-FP8-compressed-tensors.yaml deleted file mode 100644 index cca58097e8..0000000000 --- a/.buildkite/lm-eval-harness/configs/Meta-Llama-3.2-1B-Instruct-FP8-compressed-tensors.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m RedHatAI/Llama-3.2-1B-Instruct-FP8 -b "auto" -l 1319 -f 5 -t 1 -model_name: "RedHatAI/Llama-3.2-1B-Instruct-FP8" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.335 - - name: "exact_match,flexible-extract" - value: 0.323 -limit: 1319 -num_fewshot: 5 diff --git a/.buildkite/lm-eval-harness/configs/Meta-Llama-3.2-1B-Instruct-INT8-compressed-tensors.yaml b/.buildkite/lm-eval-harness/configs/Meta-Llama-3.2-1B-Instruct-INT8-compressed-tensors.yaml deleted file mode 100644 index 83e11f2be7..0000000000 --- a/.buildkite/lm-eval-harness/configs/Meta-Llama-3.2-1B-Instruct-INT8-compressed-tensors.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# For vllm script, with -t option (tensor parallel size). -# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m neuralmagic/Llama-3.2-1B-Instruct-quantized.w8a8 -b "auto" -l 1000 -f 5 -t 1 -model_name: "neuralmagic/Llama-3.2-1B-Instruct-quantized.w8a8" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.356 - - name: "exact_match,flexible-extract" - value: 0.358 -limit: 1000 -num_fewshot: 5 diff --git a/.buildkite/lm-eval-harness/configs/Meta-Llama-4-Maverick-17B-128E-Instruct-FP8-MM.yaml b/.buildkite/lm-eval-harness/configs/Meta-Llama-4-Maverick-17B-128E-Instruct-FP8-MM.yaml deleted file mode 100644 index ccb4f84201..0000000000 --- a/.buildkite/lm-eval-harness/configs/Meta-Llama-4-Maverick-17B-128E-Instruct-FP8-MM.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# For hf script, without -t option (tensor parallel size). -# bash .buildkite/lm-eval-harness/run-lm-eval-chartqa-vllm-vlm-baseline.sh -m meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 -l 100 -t 8 -model_name: "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8" -backend: "vllm-vlm" -tasks: -- name: "chartqa" - metrics: - - name: "relaxed_accuracy,none" - # TODO(zhewenl): model card is 0.90, but the actual score is 0.80. - value: 0.80 -limit: 100 -num_fewshot: 0 diff --git a/.buildkite/lm-eval-harness/configs/Meta-Llama-4-Maverick-17B-128E-Instruct-FP8.yaml b/.buildkite/lm-eval-harness/configs/Meta-Llama-4-Maverick-17B-128E-Instruct-FP8.yaml deleted file mode 100644 index 6631896ad5..0000000000 --- a/.buildkite/lm-eval-harness/configs/Meta-Llama-4-Maverick-17B-128E-Instruct-FP8.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# For hf script, without -t option (tensor parallel size). -# bash .buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh -m meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 -l 250 -t 8 -f 5 -model_name: "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8" -rocm_safetensors_load_strategy: prefetch -required_gpu_arch: - - gfx942 - - gfx950 -tasks: -- name: "mmlu_pro" - metrics: - - name: "exact_match,custom-extract" - value: 0.80 -limit: 250 # will run on 250 * 14 subjects = 3500 samples -num_fewshot: 5 -rtol: 0.05 diff --git a/.buildkite/lm-eval-harness/configs/Minitron-4B-Base-FP8.yaml b/.buildkite/lm-eval-harness/configs/Minitron-4B-Base-FP8.yaml deleted file mode 100644 index 15a836dddb..0000000000 --- a/.buildkite/lm-eval-harness/configs/Minitron-4B-Base-FP8.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# For vllm script, with -t option (tensor parallel size). -# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m mgoin/Minitron-4B-Base-FP8 -b auto -l 1000 -f 5 -t 1 -model_name: "mgoin/Minitron-4B-Base-FP8" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.231 - - name: "exact_match,flexible-extract" - value: 0.22 -limit: 1000 -num_fewshot: 5 diff --git a/.buildkite/lm-eval-harness/configs/Mixtral-8x22B-Instruct-v0.1-FP8-Dynamic.yaml b/.buildkite/lm-eval-harness/configs/Mixtral-8x22B-Instruct-v0.1-FP8-Dynamic.yaml deleted file mode 100644 index 5633a2d9b8..0000000000 --- a/.buildkite/lm-eval-harness/configs/Mixtral-8x22B-Instruct-v0.1-FP8-Dynamic.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# For vllm script, with -t option (tensor parallel size). -# bash ./run-lm-eval-gsm-vllm-baseline.sh -m neuralmagic/Mixtral-8x22B-Instruct-v0.1-FP8-dynamic -b "auto" -l 250 -f 5 -t 8 -model_name: "neuralmagic/Mixtral-8x22B-Instruct-v0.1-FP8-dynamic" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.86 - - name: "exact_match,flexible-extract" - value: 0.86 -limit: 250 -num_fewshot: 5 diff --git a/.buildkite/lm-eval-harness/configs/Mixtral-8x7B-Instruct-v0.1-FP8.yaml b/.buildkite/lm-eval-harness/configs/Mixtral-8x7B-Instruct-v0.1-FP8.yaml deleted file mode 100644 index b8024c80e8..0000000000 --- a/.buildkite/lm-eval-harness/configs/Mixtral-8x7B-Instruct-v0.1-FP8.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# For vllm script, with -t option (tensor parallel size). -# bash ./run-lm-eval-gsm-vllm-baseline.sh -m neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8 -b "auto" -l 250 -f 5 -t 4 -model_name: "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.624 - - name: "exact_match,flexible-extract" - value: 0.624 -limit: 250 -num_fewshot: 5 diff --git a/.buildkite/lm-eval-harness/configs/Mixtral-8x7B-Instruct-v0.1.yaml b/.buildkite/lm-eval-harness/configs/Mixtral-8x7B-Instruct-v0.1.yaml deleted file mode 100644 index 188a112ca3..0000000000 --- a/.buildkite/lm-eval-harness/configs/Mixtral-8x7B-Instruct-v0.1.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# For hf script, without -t option (tensor parallel size). -# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh -m neuralmagic/Mixtral-8x7B-Instruct-v0.1 -b 32 -l 250 -f 5 -model_name: "mistralai/Mixtral-8x7B-Instruct-v0.1" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.616 - - name: "exact_match,flexible-extract" - value: 0.632 -limit: 250 -num_fewshot: 5 diff --git a/.buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.yaml b/.buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.yaml deleted file mode 100644 index c1dbaef629..0000000000 --- a/.buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.yaml +++ /dev/null @@ -1,15 +0,0 @@ -model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.695 - - name: "exact_match,flexible-extract" - value: 0.447 -limit: 1319 -num_fewshot: 5 -max_model_len: 262144 -enforce_eager: false -apply_chat_template: true -fewshot_as_multiturn: true -trust_remote_code: true \ No newline at end of file diff --git a/.buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8-xpu.yaml b/.buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8-xpu.yaml deleted file mode 100644 index 72ba861195..0000000000 --- a/.buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8-xpu.yaml +++ /dev/null @@ -1,16 +0,0 @@ -model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.7142 - - name: "exact_match,flexible-extract" - value: 0.4579 -limit: 256 -num_fewshot: 5 -max_model_len: 262144 -kv_cache_dtype: fp8 -enforce_eager: true -apply_chat_template: true -fewshot_as_multiturn: true -trust_remote_code: true diff --git a/.buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8.yaml b/.buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8.yaml deleted file mode 100644 index 164733cca6..0000000000 --- a/.buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8.yaml +++ /dev/null @@ -1,17 +0,0 @@ -model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.7142 - - name: "exact_match,flexible-extract" - value: 0.4579 -moe_backend: "flashinfer_cutlass" -limit: 1319 -num_fewshot: 5 -max_model_len: 262144 -kv_cache_dtype: fp8 -enforce_eager: false -apply_chat_template: true -fewshot_as_multiturn: true -trust_remote_code: true diff --git a/.buildkite/lm-eval-harness/configs/Qwen1.5-MoE-W4A16-compressed-tensors.yaml b/.buildkite/lm-eval-harness/configs/Qwen1.5-MoE-W4A16-compressed-tensors.yaml deleted file mode 100644 index 099e0f465b..0000000000 --- a/.buildkite/lm-eval-harness/configs/Qwen1.5-MoE-W4A16-compressed-tensors.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# For vllm script, with -t option (tensor parallel size). -# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Qwen1.5-MoE-A2.7B-Chat-quantized.w4a16 -b auto -l 1319 -f 5 -t 1 -model_name: "nm-testing/Qwen1.5-MoE-A2.7B-Chat-quantized.w4a16" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.30 - - name: "exact_match,flexible-extract" - value: 0.465 -limit: 1319 -num_fewshot: 5 diff --git a/.buildkite/lm-eval-harness/configs/Qwen2-1.5B-Instruct-FP8W8.yaml b/.buildkite/lm-eval-harness/configs/Qwen2-1.5B-Instruct-FP8W8.yaml deleted file mode 100644 index 426e8ff698..0000000000 --- a/.buildkite/lm-eval-harness/configs/Qwen2-1.5B-Instruct-FP8W8.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# For vllm script, with -t option (tensor parallel size). -# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Qwen2-1.5B-Instruct-FP8W8 -b auto -l 1000 -f 5 -t 1 -model_name: "nm-testing/Qwen2-1.5B-Instruct-FP8W8" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.578 - - name: "exact_match,flexible-extract" - value: 0.585 -limit: 1000 -num_fewshot: 5 diff --git a/.buildkite/lm-eval-harness/configs/Qwen2-1.5B-Instruct-INT8-compressed-tensors.yaml b/.buildkite/lm-eval-harness/configs/Qwen2-1.5B-Instruct-INT8-compressed-tensors.yaml deleted file mode 100644 index 8d57e9dabd..0000000000 --- a/.buildkite/lm-eval-harness/configs/Qwen2-1.5B-Instruct-INT8-compressed-tensors.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# For vllm script, with -t option (tensor parallel size). -# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m neuralmagic/Qwen2-1.5B-Instruct-quantized.w8a8 -b "auto" -l 1000 -f 5 -t 1 -model_name: "neuralmagic/Qwen2-1.5B-Instruct-quantized.w8a8" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.593 - - name: "exact_match,flexible-extract" - value: 0.588 -limit: 1000 -num_fewshot: 5 diff --git a/.buildkite/lm-eval-harness/configs/Qwen2-57B-A14-Instruct.yaml b/.buildkite/lm-eval-harness/configs/Qwen2-57B-A14-Instruct.yaml deleted file mode 100644 index fc9707d0d6..0000000000 --- a/.buildkite/lm-eval-harness/configs/Qwen2-57B-A14-Instruct.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# For vllm script, with -t option (tensor parallel size). -# bash ./run-lm-eval-gsm-vllm-baseline.sh -m Qwen/Qwen2-57B-A14B-Instruct -b "auto" -l 250 -f 5 -t 4 -model_name: "Qwen/Qwen2-57B-A14B-Instruct" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.792 - - name: "exact_match,flexible-extract" - value: 0.824 -limit: 250 -num_fewshot: 5 diff --git a/.buildkite/lm-eval-harness/configs/Qwen2.5-1.5B-Instruct.yaml b/.buildkite/lm-eval-harness/configs/Qwen2.5-1.5B-Instruct.yaml deleted file mode 100644 index 54579a63a9..0000000000 --- a/.buildkite/lm-eval-harness/configs/Qwen2.5-1.5B-Instruct.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m Qwen/Qwen2.5-1.5B-Instruct -b auto -l 1319 -f 5 -t 1 -model_name: "Qwen/Qwen2.5-1.5B-Instruct" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.54 - - name: "exact_match,flexible-extract" - value: 0.59 -limit: 1319 -num_fewshot: 5 diff --git a/.buildkite/lm-eval-harness/configs/Qwen2.5-VL-3B-Instruct-FP8-dynamic.yaml b/.buildkite/lm-eval-harness/configs/Qwen2.5-VL-3B-Instruct-FP8-dynamic.yaml deleted file mode 100644 index ff43fa187b..0000000000 --- a/.buildkite/lm-eval-harness/configs/Qwen2.5-VL-3B-Instruct-FP8-dynamic.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# For vllm script, with -t option (tensor parallel size) -# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m RedHatAI/Qwen2.5-VL-3B-Instruct-FP8-Dynamic -l 1319 -t 1 -model_name: "RedHatAI/Qwen2.5-VL-3B-Instruct-FP8-Dynamic" -required_gpu_arch: - - gfx942 - - gfx950 -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.47 - - name: "exact_match,flexible-extract" - value: 0.64 -limit: 1319 -num_fewshot: 5 diff --git a/.buildkite/lm-eval-harness/configs/Qwen2.5-VL-7B-Instruct.yaml b/.buildkite/lm-eval-harness/configs/Qwen2.5-VL-7B-Instruct.yaml deleted file mode 100644 index 5f3c31743e..0000000000 --- a/.buildkite/lm-eval-harness/configs/Qwen2.5-VL-7B-Instruct.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# For vllm script, with -t option (tensor parallel size). -# bash .buildkite/lm-eval-harness/run-lm-eval-chartqa-vllm-vlm-baseline.sh -m Qwen/Qwen2.5-VL-7B-Instruct -l 2500 -t 1 - -model_name: "Qwen/Qwen2.5-VL-7B-Instruct" -backend: "vllm-vlm" -tasks: -- name: "chartqa" - metrics: - - name: "relaxed_accuracy,none" - value: 0.855 -limit: 2500 -num_fewshot: 0 diff --git a/.buildkite/lm-eval-harness/configs/Qwen3-235B-A22B-Instruct-2507-FP8.yaml b/.buildkite/lm-eval-harness/configs/Qwen3-235B-A22B-Instruct-2507-FP8.yaml deleted file mode 100644 index 84e4f3fe33..0000000000 --- a/.buildkite/lm-eval-harness/configs/Qwen3-235B-A22B-Instruct-2507-FP8.yaml +++ /dev/null @@ -1,17 +0,0 @@ -model_name: "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8" -required_gpu_arch: - - gfx942 - - gfx950 -tasks: - - name: "mmlu_pro" - metrics: - - name: "exact_match,custom-extract" - value: 0.82 -limit: 250 # will run on 250 * 14 subjects = 3500 samples -num_fewshot: 5 -enforce_eager: false # we use false to speed up the eval process -kv_cache_dtype: fp8 # we use fp8 to speed up the eval process -max_model_len: 40960 -apply_chat_template: true -fewshot_as_multiturn: true -gen_kwargs: "temperature=0,top_p=1,top_k=0,max_gen_toks=5632,until=<|ENDANSWER|>" diff --git a/.buildkite/lm-eval-harness/configs/models-large-hopper.txt b/.buildkite/lm-eval-harness/configs/models-large-hopper.txt deleted file mode 100644 index 2b6c0b5e64..0000000000 --- a/.buildkite/lm-eval-harness/configs/models-large-hopper.txt +++ /dev/null @@ -1,2 +0,0 @@ -Qwen3-235B-A22B-Instruct-2507-FP8.yaml -NVIDIA-Nemotron-3-Nano-30B-A3B-FP8.yaml diff --git a/.buildkite/lm-eval-harness/configs/models-large-rocm-tp4.txt b/.buildkite/lm-eval-harness/configs/models-large-rocm-tp4.txt deleted file mode 100644 index 4b63ccf2ca..0000000000 --- a/.buildkite/lm-eval-harness/configs/models-large-rocm-tp4.txt +++ /dev/null @@ -1,2 +0,0 @@ -Qwen3-235B-A22B-Instruct-2507-FP8.yaml -DeepSeek-V4-Flash-MXFP4.yaml diff --git a/.buildkite/lm-eval-harness/configs/models-large-rocm.txt b/.buildkite/lm-eval-harness/configs/models-large-rocm.txt deleted file mode 100644 index 4fb0b84bc4..0000000000 --- a/.buildkite/lm-eval-harness/configs/models-large-rocm.txt +++ /dev/null @@ -1 +0,0 @@ -Meta-Llama-4-Maverick-17B-128E-Instruct-FP8.yaml diff --git a/.buildkite/lm-eval-harness/configs/models-large-xpu-fp8.txt b/.buildkite/lm-eval-harness/configs/models-large-xpu-fp8.txt deleted file mode 100644 index 73b28f062c..0000000000 --- a/.buildkite/lm-eval-harness/configs/models-large-xpu-fp8.txt +++ /dev/null @@ -1 +0,0 @@ -NVIDIA-Nemotron-3-Nano-30B-A3B-FP8-xpu.yaml \ No newline at end of file diff --git a/.buildkite/lm-eval-harness/configs/models-large.txt b/.buildkite/lm-eval-harness/configs/models-large.txt deleted file mode 100644 index 385031b74f..0000000000 --- a/.buildkite/lm-eval-harness/configs/models-large.txt +++ /dev/null @@ -1,6 +0,0 @@ -Meta-Llama-3-70B-Instruct-FBGEMM-nonuniform.yaml -Meta-Llama-3-70B-Instruct.yaml -Mixtral-8x7B-Instruct-v0.1.yaml -Qwen2-57B-A14-Instruct.yaml -DeepSeek-V2-Lite-Chat.yaml -NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.yaml diff --git a/.buildkite/lm-eval-harness/configs/models-mm-large-h100.txt b/.buildkite/lm-eval-harness/configs/models-mm-large-h100.txt deleted file mode 100644 index 91e22b6459..0000000000 --- a/.buildkite/lm-eval-harness/configs/models-mm-large-h100.txt +++ /dev/null @@ -1 +0,0 @@ -Meta-Llama-4-Maverick-17B-128E-Instruct-FP8-MM.yaml diff --git a/.buildkite/lm-eval-harness/configs/models-mm-small.txt b/.buildkite/lm-eval-harness/configs/models-mm-small.txt deleted file mode 100644 index 1097d22024..0000000000 --- a/.buildkite/lm-eval-harness/configs/models-mm-small.txt +++ /dev/null @@ -1 +0,0 @@ -Qwen2.5-VL-7B-Instruct.yaml \ No newline at end of file diff --git a/.buildkite/lm-eval-harness/configs/models-small-rocm.txt b/.buildkite/lm-eval-harness/configs/models-small-rocm.txt deleted file mode 100644 index 36e0543879..0000000000 --- a/.buildkite/lm-eval-harness/configs/models-small-rocm.txt +++ /dev/null @@ -1,6 +0,0 @@ -Qwen2.5-1.5B-Instruct.yaml -Meta-Llama-3.2-1B-Instruct-INT8-compressed-tensors.yaml -Meta-Llama-3-8B-Instruct-INT8-compressed-tensors-asym.yaml -Meta-Llama-3-8B-Instruct-nonuniform-compressed-tensors.yaml -Qwen2.5-VL-3B-Instruct-FP8-dynamic.yaml -Qwen1.5-MoE-W4A16-compressed-tensors.yaml diff --git a/.buildkite/lm-eval-harness/configs/models-small.txt b/.buildkite/lm-eval-harness/configs/models-small.txt deleted file mode 100644 index 36e0543879..0000000000 --- a/.buildkite/lm-eval-harness/configs/models-small.txt +++ /dev/null @@ -1,6 +0,0 @@ -Qwen2.5-1.5B-Instruct.yaml -Meta-Llama-3.2-1B-Instruct-INT8-compressed-tensors.yaml -Meta-Llama-3-8B-Instruct-INT8-compressed-tensors-asym.yaml -Meta-Llama-3-8B-Instruct-nonuniform-compressed-tensors.yaml -Qwen2.5-VL-3B-Instruct-FP8-dynamic.yaml -Qwen1.5-MoE-W4A16-compressed-tensors.yaml diff --git a/.buildkite/lm-eval-harness/conftest.py b/.buildkite/lm-eval-harness/conftest.py deleted file mode 100644 index c0d60dd532..0000000000 --- a/.buildkite/lm-eval-harness/conftest.py +++ /dev/null @@ -1,44 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from pathlib import Path - -import pytest - - -def pytest_addoption(parser): - parser.addoption( - "--config-list-file", - action="store", - help="Path to the file listing model config YAMLs (one per line)", - ) - parser.addoption( - "--tp-size", - action="store", - default="1", - help="Tensor parallel size to use for evaluation", - ) - - -@pytest.fixture(scope="session") -def config_list_file(pytestconfig, config_dir): - rel_path = pytestconfig.getoption("--config-list-file") - return config_dir / rel_path - - -@pytest.fixture(scope="session") -def tp_size(pytestconfig): - return pytestconfig.getoption("--tp-size") - - -def pytest_generate_tests(metafunc): - if "config_filename" in metafunc.fixturenames: - rel_path = metafunc.config.getoption("--config-list-file") - config_list_file = Path(rel_path).resolve() - config_dir = config_list_file.parent - with open(config_list_file, encoding="utf-8") as f: - configs = [ - config_dir / line.strip() - for line in f - if line.strip() and not line.startswith("#") - ] - metafunc.parametrize("config_filename", configs) diff --git a/.buildkite/lm-eval-harness/run-lm-eval-chartqa-vllm-vlm-baseline.sh b/.buildkite/lm-eval-harness/run-lm-eval-chartqa-vllm-vlm-baseline.sh deleted file mode 100755 index b495c0d123..0000000000 --- a/.buildkite/lm-eval-harness/run-lm-eval-chartqa-vllm-vlm-baseline.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -# We can use this script to compute baseline accuracy on chartqa for vllm. -# -# Make sure you have lm-eval-harness installed: -# pip install "lm-eval[api]>=0.4.12" - -usage() { - echo`` - echo "Runs lm eval harness on ChartQA using multimodal vllm." - echo "This pathway is intended to be used to create baselines for " - echo "our correctness tests in vllm's CI." - echo - echo "usage: ${0} " - echo - echo " -m - huggingface stub or local directory of the model" - echo " -l - limit number of samples to run" - echo " -t - tensor parallel size to run at" - echo -} - -while getopts "m:l:t:" OPT; do - case ${OPT} in - m ) - MODEL="$OPTARG" - ;; - l ) - LIMIT="$OPTARG" - ;; - t ) - TP_SIZE="$OPTARG" - ;; - \? ) - usage - exit 1 - ;; - esac -done - -lm_eval --model vllm-vlm \ - --model_args "pretrained=$MODEL,tensor_parallel_size=$TP_SIZE" \ - --tasks chartqa \ - --batch_size auto \ - --apply_chat_template \ - --limit "$LIMIT" diff --git a/.buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh b/.buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh deleted file mode 100755 index e430e6183b..0000000000 --- a/.buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -# We can use this script to compute baseline accuracy on GSM for transformers. -# -# Make sure you have lm-eval-harness installed: -# pip install "lm-eval[api]>=0.4.12" - -usage() { - echo`` - echo "Runs lm eval harness on GSM8k using huggingface transformers." - echo "This pathway is intended to be used to create baselines for " - echo "our automated nm-test-accuracy workflow" - echo - echo "usage: ${0} " - echo - echo " -m - huggingface stub or local directory of the model" - echo " -b - batch size to run the evaluation at" - echo " -l - limit number of samples to run" - echo " -f - number of fewshot samples to use" - echo -} - -while getopts "m:b:l:f:" OPT; do - case ${OPT} in - m ) - MODEL="$OPTARG" - ;; - b ) - BATCH_SIZE="$OPTARG" - ;; - l ) - LIMIT="$OPTARG" - ;; - f ) - FEWSHOT="$OPTARG" - ;; - \? ) - usage - exit 1 - ;; - esac -done - -lm_eval --model hf \ - --model_args "pretrained=$MODEL,parallelize=True" \ - --tasks gsm8k --num_fewshot "$FEWSHOT" --limit "$LIMIT" \ - --batch_size "$BATCH_SIZE" diff --git a/.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh b/.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh deleted file mode 100644 index f1a541ddbe..0000000000 --- a/.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -# We can use this script to compute baseline accuracy on GSM for vllm. -# We use this for fp8, which HF does not support. -# -# Make sure you have lm-eval-harness installed: -# pip install "lm-eval[api]>=0.4.12" - -usage() { - echo`` - echo "Runs lm eval harness on GSM8k using huggingface transformers." - echo "This pathway is intended to be used to create baselines for " - echo "our automated nm-test-accuracy workflow" - echo - echo "usage: ${0} " - echo - echo " -m - huggingface stub or local directory of the model" - echo " -b - batch size to run the evaluation at" - echo " -l - limit number of samples to run" - echo " -f - number of fewshot samples to use" - echo " -t - tensor parallel size to run at" - echo -} - -while getopts "m:b:l:f:t:" OPT; do - case ${OPT} in - m ) - MODEL="$OPTARG" - ;; - b ) - BATCH_SIZE="$OPTARG" - ;; - l ) - LIMIT="$OPTARG" - ;; - f ) - FEWSHOT="$OPTARG" - ;; - t ) - TP_SIZE="$OPTARG" - ;; - \? ) - usage - exit 1 - ;; - esac -done - -lm_eval --model vllm \ - --model_args "pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,add_bos_token=true,trust_remote_code=true,max_model_len=4096" \ - --tasks gsm8k --num_fewshot "$FEWSHOT" --limit "$LIMIT" \ - --batch_size "$BATCH_SIZE" diff --git a/.buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh b/.buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh deleted file mode 100644 index ba8da9fc3f..0000000000 --- a/.buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -# We can use this script to compute baseline accuracy on MMLUPRO for vllm. -# We use this for fp8, which HF does not support. -# -# Make sure you have lm-eval-harness installed: -# pip install "lm-eval[api]>=0.4.12" - -usage() { - echo`` - echo "Runs lm eval harness on MMLU Pro using huggingface transformers." - echo "This pathway is intended to be used to create baselines for " - echo "our automated nm-test-accuracy workflow" - echo - echo "usage: ${0} " - echo - echo " -m - huggingface stub or local directory of the model" - echo " -l - limit number of samples to run" - echo " -f - number of fewshot samples to use" - echo " -t - tensor parallel size to run at" - echo -} - -while getopts "m:l:f:t:" OPT; do - case ${OPT} in - m ) - MODEL="$OPTARG" - ;; - l ) - LIMIT="$OPTARG" - ;; - f ) - FEWSHOT="$OPTARG" - ;; - t ) - TP_SIZE="$OPTARG" - ;; - \? ) - usage - exit 1 - ;; - esac -done - -lm_eval --model vllm \ - --model_args "pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,add_bos_token=true,trust_remote_code=true,max_model_len=4096" \ - --tasks mmlu_pro --num_fewshot "$FEWSHOT" --limit "$LIMIT" \ - --batch_size auto diff --git a/.buildkite/lm-eval-harness/test_lm_eval_correctness.py b/.buildkite/lm-eval-harness/test_lm_eval_correctness.py deleted file mode 100644 index e4c99adb7b..0000000000 --- a/.buildkite/lm-eval-harness/test_lm_eval_correctness.py +++ /dev/null @@ -1,221 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""LM eval harness on model to compare vs HF baseline computed offline. -Configs are found in configs/$MODEL.yaml - -pytest -s -v test_lm_eval_correctness.py \ - --config-list-file=configs/models-small.txt \ - --tp-size=1 -""" - -import gc -import os -import time -from contextlib import contextmanager - -import lm_eval -import pytest -import torch -import yaml -from lm_eval.api.registry import get_model -from lm_eval.utils import simple_parse_args_string - -from vllm import LLM -from vllm.distributed import cleanup_dist_env_and_memory -from vllm.logger import init_logger -from vllm.platforms import current_platform - -logger = init_logger(__name__) - -DEFAULT_RTOL = 0.08 -ROCM_ENGINE_SHUTDOWN_TIMEOUT_S = 60.0 -MEMORY_RELEASE_TIMEOUT_S = 240.0 - - -@contextmanager -def scoped_env_vars(new_env: dict[str, str]): - if not new_env: - # Fast path: nothing to do - yield - return - - old_values = {} - new_keys = [] - - try: - for key, value in new_env.items(): - if key in os.environ: - old_values[key] = os.environ[key] - else: - new_keys.append(key) - os.environ[key] = str(value) - yield - finally: - # Restore / clean up - for key, value in old_values.items(): - os.environ[key] = value - for key in new_keys: - os.environ.pop(key, None) - - -def _wait_for_memory_release(gpu_memory_utilization: float) -> None: - devices = range(torch.accelerator.device_count()) - deadline = time.monotonic() + MEMORY_RELEASE_TIMEOUT_S - while True: - memory = [torch.accelerator.get_memory_info(d) for d in devices] - if all(free >= total * gpu_memory_utilization for free, total in memory): - return - if time.monotonic() >= deadline: - usage = ", ".join( - f"{d}: {free / 2**30:.2f}/{total / 2**30:.2f} GiB free" - for d, (free, total) in zip(devices, memory) - ) - logger.warning( - "GPU memory not released after %ss (%s); the next model needs %.2f", - MEMORY_RELEASE_TIMEOUT_S, - usage, - gpu_memory_utilization, - ) - return - time.sleep(1) - - -def _shutdown_lm(lm) -> None: - if not current_platform.is_rocm(): - return - llm = getattr(lm, "model", None) - if not isinstance(llm, LLM): - return - gpu_memory_utilization = ( - llm.llm_engine.vllm_config.cache_config.gpu_memory_utilization - ) - try: - llm.llm_engine.engine_core.shutdown(timeout=ROCM_ENGINE_SHUTDOWN_TIMEOUT_S) - except Exception: - logger.exception("Engine core shutdown raised; GPU memory may leak") - del llm - lm.model = None - gc.collect() - cleanup_dist_env_and_memory() - _wait_for_memory_release(gpu_memory_utilization) - - -def launch_lm_eval(eval_config, tp_size): - trust_remote_code = eval_config.get("trust_remote_code", False) - max_model_len = eval_config.get("max_model_len", 4096) - batch_size = eval_config.get("batch_size", "auto") - backend = eval_config.get("backend", "vllm") - enforce_eager = eval_config.get("enforce_eager", "true") - kv_cache_dtype = eval_config.get("kv_cache_dtype", "auto") - model_args = ( - f"pretrained={eval_config['model_name']}," - f"tensor_parallel_size={tp_size}," - f"enforce_eager={enforce_eager}," - f"kv_cache_dtype={kv_cache_dtype}," - f"add_bos_token=true," - f"trust_remote_code={trust_remote_code}," - f"max_model_len={max_model_len}," - "allow_deprecated_quantization=True," - ) - - if current_platform.is_rocm() and "Nemotron-3" in eval_config["model_name"]: - model_args += "attention_backend=TRITON_ATTN" - - moe_backend = eval_config.get("moe_backend", None) - if moe_backend is not None: - model_args += f"moe_backend={moe_backend}," - - if current_platform.is_rocm(): - rocm_load_strategy = eval_config.get("rocm_safetensors_load_strategy") - if rocm_load_strategy is not None: - model_args += f"safetensors_load_strategy={rocm_load_strategy}," - - tokenizer_mode = eval_config.get("tokenizer_mode", None) - if tokenizer_mode is not None: - model_args += f"tokenizer_mode={tokenizer_mode}," - - env_vars = eval_config.get("env_vars", None) - with scoped_env_vars(env_vars): - if current_platform.is_rocm(): - model = get_model(backend).create_from_arg_string( - model_args, - {"batch_size": batch_size, "max_batch_size": None, "device": None}, - ) - model_kwargs = {"metadata": simple_parse_args_string(model_args)} - else: - model = backend - model_kwargs = {"model_args": model_args} - results = lm_eval.simple_evaluate( - model=model, - tasks=[task["name"] for task in eval_config["tasks"]], - num_fewshot=eval_config["num_fewshot"], - limit=eval_config["limit"], - # TODO(yeq): using chat template w/ fewshot_as_multiturn is supposed help - # text models. however, this is regressing measured strict-match for - # existing text models in CI, so only apply it for mm, or explicitly set - apply_chat_template=eval_config.get( - "apply_chat_template", backend == "vllm-vlm" - ), - fewshot_as_multiturn=eval_config.get("fewshot_as_multiturn", False), - # Forward decoding and early-stop controls (e.g., max_gen_toks, until=...) - gen_kwargs=eval_config.get("gen_kwargs"), - batch_size=batch_size, - **model_kwargs, - ) - _shutdown_lm(model) - return results - - -def _check_rocm_gpu_arch_requirement(eval_config): - """Skip the test if the model requires a ROCm GPU arch not present. - - Model YAML configs can specify:: - - required_gpu_arch: - - gfx942 - - gfx950 - - The check only applies on ROCm. On other platforms (e.g. CUDA) the - field is ignored so that shared config files work for both NVIDIA and - AMD CI pipelines. - """ - required_archs = eval_config.get("required_gpu_arch") - if not required_archs: - return - - if not current_platform.is_rocm(): - return - - from vllm.platforms.rocm import _GCN_ARCH # noqa: E402 - - if not any(arch in _GCN_ARCH for arch in required_archs): - pytest.skip( - f"Model requires GPU arch {required_archs}, " - f"but detected arch is '{_GCN_ARCH}'" - ) - - -def test_lm_eval_correctness_param(config_filename, tp_size): - eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8")) - - _check_rocm_gpu_arch_requirement(eval_config) - - results = launch_lm_eval(eval_config, tp_size) - - rtol = eval_config.get("rtol", DEFAULT_RTOL) - - success = True - for task in eval_config["tasks"]: - for metric in task["metrics"]: - ground_truth = metric["value"] - measured_value = results["results"][task["name"]][metric["name"]] - print( - f"{task['name']} | {metric['name']}: " - f"ground_truth={ground_truth:.3f} | " - f"measured={measured_value:.3f} | rtol={rtol}" - ) - - min_acceptable = ground_truth * (1 - rtol) - success = success and measured_value >= min_acceptable - - assert success diff --git a/.buildkite/performance-benchmarks/README.md b/.buildkite/performance-benchmarks/README.md deleted file mode 100644 index 3a321c0fef..0000000000 --- a/.buildkite/performance-benchmarks/README.md +++ /dev/null @@ -1,180 +0,0 @@ -# vLLM benchmark suite - -## Introduction - -This directory contains a benchmarking suite for **developers** to run locally and gain clarity on whether their PR improves/degrades vllm's performance. -vLLM also maintains a continuous performance benchmark under [perf.vllm.ai](https://perf.vllm.ai/), hosted under PyTorch CI HUD. - -## Performance benchmark quick overview - -**Benchmarking Coverage**: latency, throughput and fix-qps serving on B200, A100, H100, Intel® Xeon® Processors, Intel® Gaudi® 3 Accelerators and Arm® Neoverse™ with different models. - -**Benchmarking Duration**: about 1hr. - -**For benchmarking developers**: please try your best to constraint the duration of benchmarking to about 1 hr so that it won't take forever to run. - -## Trigger the benchmark - -The benchmark needs to be triggered manually: - -```bash -bash .buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh -``` - -Runtime environment variables: - -- `ON_CPU`: set the value to '1' on Intel® Xeon® and Arm® Neoverse™ Processors. Default value is 0. -- `SERVING_JSON`: JSON file to use for the serving tests. Default value is empty string (use default file). -- `LATENCY_JSON`: JSON file to use for the latency tests. Default value is empty string (use default file). -- `THROUGHPUT_JSON`: JSON file to use for the throughout tests. Default value is empty string (use default file). -- `REMOTE_HOST`: IP for the remote vLLM service to benchmark. Default value is empty string. -- `REMOTE_PORT`: Port for the remote vLLM service to benchmark. Default value is empty string. - -## Performance benchmark details - -See [performance-benchmarks-descriptions.md](performance-benchmarks-descriptions.md) for detailed descriptions, and use `tests/latency-tests.json`, `tests/throughput-tests.json`, `tests/serving-tests.json` to configure the test cases. -> NOTE: For Intel® Xeon® Processors, use `tests/latency-tests-cpu.json`, `tests/throughput-tests-cpu.json`, `tests/serving-tests-cpu.json` instead. -> For Intel® Gaudi® 3 Accelerators, use `tests/latency-tests-hpu.json`, `tests/throughput-tests-hpu.json`, `tests/serving-tests-hpu.json` instead. -> For Arm® Neoverse™, use `tests/latency-tests-arm64-cpu.json`, `tests/throughput-tests-arm64-cpu.json`, `tests/serving-tests-arm64-cpu.json` instead. - -### Latency test - -Here is an example of one test inside `latency-tests.json`: - -```json -[ - { - "test_name": "latency_llama8B_tp1", - "parameters": { - "model": "meta-llama/Meta-Llama-3-8B", - "tensor_parallel_size": 1, - "load_format": "dummy", - "num_iters_warmup": 5, - "num_iters": 15 - } - }, -] -``` - -In this example: - -- The `test_name` attributes is a unique identifier for the test. In `latency-tests.json`, it must start with `latency_`. -- The `parameters` attribute control the command line arguments to be used for `vllm bench latency`. Note that please use underline `_` instead of the dash `-` when specifying the command line arguments, and `run-performance-benchmarks.sh` will convert the underline to dash when feeding the arguments to `vllm bench latency`. For example, the corresponding command line arguments for `vllm bench latency` will be `--model meta-llama/Meta-Llama-3-8B --tensor-parallel-size 1 --load-format dummy --num-iters-warmup 5 --num-iters 15` - -Note that the performance numbers are highly sensitive to the value of the parameters. Please make sure the parameters are set correctly. - -WARNING: The benchmarking script will save json results by itself, so please do not configure `--output-json` parameter in the json file. - -### Throughput test - -The tests are specified in `throughput-tests.json`. The syntax is similar to `latency-tests.json`, except for that the parameters will be fed forward to `vllm bench throughput`. - -The number of this test is also stable -- a slight change on the value of this number might vary the performance numbers by a lot. - -### Serving test - -We test the throughput by using `vllm bench serve` with request rate = inf to cover the online serving overhead. The corresponding parameters are in `serving-tests.json`, and here is an example: - -```json -[ - { - "test_name": "serving_llama8B_tp1_sharegpt", - "qps_list": [1, 4, 16, "inf"], - "server_parameters": { - "model": "meta-llama/Meta-Llama-3-8B", - "tensor_parallel_size": 1, - "disable_log_stats": "", - "load_format": "dummy" - }, - "client_parameters": { - "model": "meta-llama/Meta-Llama-3-8B", - "backend": "vllm", - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "num_prompts": 200 - } - }, -] -``` - -Inside this example: - -- The `test_name` attribute is also a unique identifier for the test. It must start with `serving_`. -- The `server-parameters` includes the command line arguments for vLLM server. -- The `client-parameters` includes the command line arguments for `vllm bench serve`. -- The `qps_list` controls the list of qps for test. It will be used to configure the `--request-rate` parameter in `vllm bench serve` - -The number of this test is less stable compared to the delay and latency benchmarks (due to randomized sharegpt dataset sampling inside `benchmark_serving.py`), but a large change on this number (e.g. 5% change) still vary the output greatly. - -WARNING: The benchmarking script will save json results by itself, so please do not configure `--save-results` or other results-saving-related parameters in `serving-tests.json`. - -#### Default Parameters Field - -We can specify default parameters in a JSON field with key `defaults`. Parameters defined in the field are applied globally to all serving tests, and can be overridden in test case fields. Here is an example: - -
- An Example of default parameters field - -```json -{ - "defaults": { - "qps_list": [ - "inf" - ], - "server_environment_variables": { - "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1 - }, - "server_parameters": { - "tensor_parallel_size": 1, - "dtype": "bfloat16", - "block_size": 128, - "disable_log_stats": "", - "load_format": "dummy" - }, - "client_parameters": { - "backend": "vllm", - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128, - "num_prompts": 200, - "ignore-eos": "" - } - }, - "tests": [ - { - "test_name": "serving_llama3B_tp2_random_128_128", - "server_parameters": { - "model": "meta-llama/Llama-3.2-3B-Instruct", - "tensor_parallel_size": 2, - }, - "client_parameters": { - "model": "meta-llama/Llama-3.2-3B-Instruct", - } - }, - { - "test_name": "serving_qwen3_tp4_random_128_128", - "server_parameters": { - "model": "Qwen/Qwen3-14B", - "tensor_parallel_size": 4, - }, - "client_parameters": { - "model": "Qwen/Qwen3-14B", - } - }, - ] -} -``` - -
- -### Visualizing the results - -The `convert-results-json-to-markdown.py` helps you put the benchmarking results inside a markdown table, by formatting [descriptions.md](performance-benchmarks-descriptions.md) with real benchmarking results. -You can find the result presented as a table inside the `buildkite/performance-benchmark` job page. -If you do not see the table, please wait till the benchmark finish running. -The json version of the table (together with the json version of the benchmark) will be also attached to the markdown file. -The raw benchmarking results (in the format of json files) are in the `Artifacts` tab of the benchmarking. - -#### Performance Results Comparison - -Follow the instructions in [performance results comparison](https://docs.vllm.ai/en/latest/benchmarking/dashboard/#performance-results-comparison) to analyze performance results and the sizing guide. diff --git a/.buildkite/performance-benchmarks/performance-benchmarks-descriptions.md b/.buildkite/performance-benchmarks/performance-benchmarks-descriptions.md deleted file mode 100644 index b9437ac5ca..0000000000 --- a/.buildkite/performance-benchmarks/performance-benchmarks-descriptions.md +++ /dev/null @@ -1,65 +0,0 @@ -# Performance benchmarks descriptions - -## Latency tests - -- Input length: 32 tokens. -- Output length: 128 tokens. -- Batch size: fixed (8). -- GPU/HPU Models: llama-3.1 8B, llama-3 70B, mixtral 8x7B. -- CPU Models: llama-3.1 8B. -- Evaluation metrics: end-to-end latency (mean, median, p99). - -{latency_tests_markdown_table} - -## Throughput tests - -- Input length: randomly sample 200 prompts from ShareGPT dataset (with fixed random seed). -- Output length: the corresponding output length of these 200 prompts. -- Batch size: dynamically determined by vllm to achieve maximum throughput. -- GPU/HPU Models: llama-3.1 8B, llama-3 70B, mixtral 8x7B. -- CPU Models: llama-3.1 8B. -- Evaluation metrics: throughput. - -{throughput_tests_markdown_table} - -## Serving tests - -- Input length: randomly sample 200 prompts from ShareGPT dataset (with fixed random seed). -- Output length: the corresponding output length of these 200 prompts. -- Batch size: dynamically determined by vllm and the arrival pattern of the requests. -- **Average QPS (query per second)**: 1, 4, 16 and inf. QPS = inf means all requests come at once. For other QPS values, the arrival time of each query is determined using a random Poisson process (with fixed random seed). -- GPU/HPU Models: llama-3.1 8B, llama-3 70B, mixtral 8x7B. -- We also added a speculative decoding test for llama-3 70B on GPU, under QPS 2 -- CPU Models: llama-3.1 8B. -- Evaluation metrics: throughput, TTFT (time to the first token, with mean, median and p99), ITL (inter-token latency, with mean, median and p99). -- For CPU, we added random dataset tests to benchmark fixed input/output length with 100 prompts. - -{serving_tests_markdown_table} - -## Platform Information - -{platform_markdown_table} - -## json version of the benchmarking tables - -This section contains the data of the markdown tables above in JSON format. -You can load the benchmarking tables into pandas dataframes as follows: - -```python -import json -import pandas as pd - -benchmarking_results_json = """The json string""" -benchmarking_results = json.loads(benchmarking_results_json) -latency_results = pd.DataFrame.from_dict(benchmarking_results["latency"]) -throughput_results = pd.DataFrame.from_dict(benchmarking_results["throughput"]) -serving_results = pd.DataFrame.from_dict(benchmarking_results["serving"]) -``` - -The json string for all benchmarking tables: - -```json -{benchmarking_results_in_json_string} -``` - -You can also check the raw experiment data in the Artifact tab of the Buildkite page. diff --git a/.buildkite/performance-benchmarks/scripts/compare-json-results.py b/.buildkite/performance-benchmarks/scripts/compare-json-results.py deleted file mode 100644 index 823c7488ec..0000000000 --- a/.buildkite/performance-benchmarks/scripts/compare-json-results.py +++ /dev/null @@ -1,1325 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from __future__ import annotations - -import argparse -import html as _html -import json -import os -from contextlib import nullcontext -from dataclasses import dataclass -from importlib import util -from pathlib import Path - -import pandas as pd - -pd.options.display.float_format = "{:.2f}".format -plotly_found = util.find_spec("plotly.express") is not None - -DEFAULT_INFO_COLS = [ - "Model", - "Dataset Name", - "Input Len", - "Output Len", - # "TP Size", - # "PP Size", - "# of max concurrency.", - "qps", -] - -# Safety net: if any DataFrame leaks into to_html(), keep precision at 2. -pd.set_option("display.precision", 2) -pd.set_option("display.float_format", lambda x: f"{x:.2f}") - - -# ----------------------------- -# Concurrency normalization (NEW, small) -# ----------------------------- -def _find_concurrency_col(df: pd.DataFrame) -> str: - for c in [ - "# of max concurrency.", - "# of max concurrency", - "Max Concurrency", - "max_concurrency", - "Concurrency", - ]: - if c in df.columns: - return c - - for c in df.columns: - if "concurr" in str(c).lower(): - s = df[c] - if s.dtype.kind in "iu" and s.nunique() > 1 and s.min() >= 1: - return c - - raise ValueError( - "Cannot infer concurrency column. " - "Please rename the column to one of the known names " - "or add an explicit override (e.g., --concurrency-col)." - ) - - -def _normalize_concurrency_in_df( - df: pd.DataFrame, canonical: str = "# of max concurrency." -) -> pd.DataFrame: - if canonical in df.columns: - return df - detected = _find_concurrency_col(df) - if detected in df.columns and detected != canonical: - return df.rename(columns={detected: canonical}) - df[canonical] = pd.NA - return df - - -# ----------------------------- -# Core data compare -# ----------------------------- -def compare_data_columns( - files: list[str], - name_column: str, - data_column: str, - info_cols: list[str], - drop_column: str, - debug: bool = False, -): - """Align concatenation by keys derived from info_cols instead of row order. - - Pick one canonical key list: subset of info_cols present in ALL files. - - For each file: set index to those keys, aggregate duplicates - (mean for metric, first for names). - - Concat along axis=1 (indexes align), then reset_index so callers can - group by columns. - - If --debug, add a _name column per file. - - Minimal fix to support different max_concurrency lists across files: - - normalize concurrency column naming to "# of max concurrency." - - align on UNION of keys (missing points become NaN) - - BUGFIX: don't drop throughput rows based on P99/Median presence - """ - print("\ncompare_data_column:", data_column) - - frames = [] - raw_data_cols: list[str] = [] - - # Determine key cols after normalizing concurrency - cols_per_file: list[set] = [] - for f in files: - try: - df_tmp = pd.read_json(f, orient="records") - except Exception as err: - raise ValueError(f"Failed to read {f}") from err - df_tmp = _normalize_concurrency_in_df(df_tmp, canonical="# of max concurrency.") - cols_per_file.append(set(df_tmp.columns)) - - key_cols = [c for c in info_cols if all(c in cset for cset in cols_per_file)] - if not key_cols: - key_cols = [c for c in info_cols if c in list(cols_per_file[0])] - if not key_cols: - raise ValueError( - "No common key columns found from info_cols across the input files." - ) - - union_index = None - metas: list[pd.DataFrame] = [] - staged: list[tuple[str, pd.Series, pd.Series | None]] = [] - - for file in files: - df = pd.read_json(file, orient="records") - df = _normalize_concurrency_in_df(df, canonical="# of max concurrency.") - - # BUGFIX: only drop rows for latency-like metrics; throughput rows may have - # NaN in P99/Median columns even if the column exists in the JSON. - metric_lc = str(data_column).lower() - is_latency_metric = ( - "ttft" in metric_lc - or "tpot" in metric_lc - or "p99" in metric_lc - or "median" in metric_lc - or metric_lc.strip() in {"p99", "median"} - ) - if is_latency_metric and drop_column in df.columns: - df = df.dropna(subset=[drop_column], ignore_index=True) - - for c in ( - "Input Len", - "Output Len", - "TP Size", - "PP Size", - "# of max concurrency.", - "qps", - ): - if c in df.columns: - df[c] = pd.to_numeric(df[c], errors="coerce") - - for c in key_cols: - if c not in df.columns: - df[c] = pd.NA - - df_idx = df.set_index(key_cols, drop=False) - - meta = df_idx[key_cols] - if not meta.index.is_unique: - meta = meta.groupby(level=key_cols, dropna=False).first() - - file_label = "/".join(file.split("/")[:-1]) or os.path.basename(file) - - if data_column in df_idx.columns: - s = df_idx[data_column] - if not s.index.is_unique: - s = s.groupby(level=key_cols, dropna=False).mean() - else: - # keep NA series to preserve meta keys for union_index - s = pd.Series(pd.NA, index=meta.index) - s.name = file_label - - name_s = None - if debug and name_column in df_idx.columns: - name_s = df_idx[name_column] - if not name_s.index.is_unique: - name_s = name_s.groupby(level=key_cols, dropna=False).first() - name_s.name = f"{file_label}_name" - - if union_index is None: - union_index = meta.index - else: - union_index = union_index.union(meta.index) - metas.append(meta) - - staged.append((file_label, s, name_s)) - - if union_index is None: - raise ValueError("No data found after loading inputs.") - - # meta first (union-aligned): build UNION meta across all files - if metas: - meta_union = pd.concat(metas, axis=0) - # Collapse duplicates on the MultiIndex; keep first non-null per column - meta_union = meta_union.groupby(level=key_cols, dropna=False).first() - frames.append(meta_union.reindex(union_index)) - - # values + ratios (union-aligned) - metric_series_aligned: list[pd.Series] = [] - for file_label, s, name_s in staged: - s_aligned = s.reindex(union_index) - frames.append(s_aligned) - raw_data_cols.append(file_label) - metric_series_aligned.append(s_aligned) - - if debug and name_s is not None: - frames.append(name_s.reindex(union_index)) - - if len(metric_series_aligned) >= 2: - base = metric_series_aligned[0] - current = metric_series_aligned[-1] - if "P99" in str(data_column) or "Median" in str(data_column): - ratio = base / current - else: - ratio = current / base - ratio = ratio.mask(base == 0) - ratio.name = f"Ratio 1 vs {len(metric_series_aligned)}" - frames.append(ratio) - - concat_df = pd.concat(frames, axis=1).reset_index(drop=True) - - front = [c for c in info_cols if c in concat_df.columns] - rest = [c for c in concat_df.columns if c not in front] - concat_df = concat_df[front + rest] - - print(raw_data_cols) - return concat_df, raw_data_cols - - -# ----------------------------- -# Split helper -# ----------------------------- -def split_json_by_tp_pp( - input_file: str = "benchmark_results.json", output_root: str = "." -) -> list[str]: - with open(input_file, encoding="utf-8") as f: - data = json.load(f) - - if isinstance(data, dict): - for key in ("results", "serving_results", "benchmarks", "data"): - if isinstance(data.get(key), list): - data = data[key] - break - - df = pd.DataFrame(data) - - name_col = next( - (c for c in ["Test name", "test_name", "Test Name"] if c in df.columns), None - ) - if name_col: - df = df[ - df[name_col].astype(str).str.contains(r"serving", case=False, na=False) - ].copy() - - rename_map = { - "tp_size": "TP Size", - "tensor_parallel_size": "TP Size", - "pp_size": "PP Size", - "pipeline_parallel_size": "PP Size", - } - df.rename( - columns={k: v for k, v in rename_map.items() if k in df.columns}, inplace=True - ) - - if "TP Size" not in df.columns: - df["TP Size"] = 1 - if "PP Size" not in df.columns: - df["PP Size"] = 1 - - df["TP Size"] = pd.to_numeric(df["TP Size"], errors="coerce").fillna(1).astype(int) - df["PP Size"] = pd.to_numeric(df["PP Size"], errors="coerce").fillna(1).astype(int) - - saved_paths: list[str] = [] - for (tp, pp), group_df in df.groupby(["TP Size", "PP Size"], dropna=False): - folder_name = os.path.join(output_root, f"tp{int(tp)}_pp{int(pp)}") - os.makedirs(folder_name, exist_ok=True) - filepath = os.path.join(folder_name, "benchmark_results.json") - group_df.to_json(filepath, orient="records", indent=2, force_ascii=False) - print(f"Saved: {filepath}") - saved_paths.append(filepath) - - return saved_paths - - -# ----------------------------- -# Styling helpers -# ----------------------------- -def _highlight_threshold( - df: pd.DataFrame, - threshold: float, - slack_pct: float = 0.0, -) -> pd.io.formats.style.Styler: - conc_col = _find_concurrency_col(df) - key_cols = [ - c - for c in ["Model", "Dataset Name", "Input Len", "Output Len", conc_col] - if c in df.columns - ] - conf_cols = [ - c for c in df.columns if c not in key_cols and not str(c).startswith("Ratio") - ] - conf_cols = [c for c in conf_cols if pd.api.types.is_numeric_dtype(df[c])] - - try: - slack_pct = float(slack_pct or 0.0) - except Exception: - slack_pct = 0.0 - slack_limit = threshold * (1.0 + slack_pct / 100.0) - - def _cell(v): - if pd.isna(v): - return "" - if v <= threshold: - # Strict SLA - return "background-color:#e6ffe6;font-weight:bold;" - if v <= slack_limit: - # Within slack range - return "background-color:#ffe5cc;font-weight:bold;" - return "" - - return df.style.map(_cell, subset=conf_cols) - - -def highlight_ratio_columns(styler: pd.io.formats.style.Styler): - ratio_cols = [c for c in styler.data.columns if "ratio" in str(c).lower()] - if not ratio_cols: - return styler - - styler = styler.apply( - lambda _: ["background-color: #fff3b0"] * len(styler.data), - subset=ratio_cols, - axis=0, - ) - - styler = styler.set_table_styles( - [ - { - "selector": f"th.col_heading.level0.col{i}", - "props": [("background-color", "#fff3b0")], - } - for i, col in enumerate(styler.data.columns) - if col in ratio_cols - ], - overwrite=False, - ) - return styler - - -def _apply_two_decimals( - styler: pd.io.formats.style.Styler, -) -> pd.io.formats.style.Styler: - df = styler.data - num_cols = df.select_dtypes("number").columns - if len(num_cols) == 0: - return styler - return styler.format({c: "{:.2f}" for c in num_cols}, na_rep="") - - -# ----------------------------- -# Export helpers (Excel + CSV) -# ----------------------------- -def _sanitize_sheet_name(name: str) -> str: - """Excel sheet constraints: - - max 31 chars - - cannot contain: : \ / ? * [ ] - - cannot be empty - - NOTE: Use fast, non-regex operations here to avoid the third-party `regex` - module's compile overhead/edge-cases on some systems. - """ - name = "sheet" if name is None else str(name) - - # Replace illegal characters with underscore. - trans = str.maketrans( - { - ":": "_", - "\\": "_", - "/": "_", - "?": "_", - "*": "_", - "[": "_", - "]": "_", - } - ) - name = name.translate(trans) - - # Strip quotes/spaces and collapse whitespace. - name = name.strip().strip("'") - name = " ".join(name.split()) - - if not name: - name = "sheet" - return name[:31] - - -def _group_to_sheet_base(group_cols: list[str], gkey_tuple) -> str: - d = dict(zip(group_cols, gkey_tuple)) - - # Always keep input/output lengths (these are important). - ilen = d.get("Input Len", "") - olen = d.get("Output Len", "") - lens = f"_{ilen}x{olen}" if ilen != "" and olen != "" else "" - - # Shorten model name aggressively to make room for lens. - model = d.get("Model", "model") - leaf = str(model).split("/")[-1] - - max_model_len = max(1, 31 - len(lens)) - model_short = leaf[:max_model_len] - - return _sanitize_sheet_name(f"{model_short}{lens}") - - -def _write_tables_to_excel_sheet( - writer: pd.ExcelWriter, sheet: str, blocks: list[tuple[str, pd.DataFrame]] -): - """Write all blocks to a sheet with a single to_excel() call. - - Pandas+openpyxl can be extremely slow when called many times per sheet. - We flatten blocks into one table with a 'Section' column to keep structure - while making Excel generation fast and deterministic. - """ - if not blocks: - pd.DataFrame().to_excel(writer, sheet_name=sheet, index=False) - return - - combined_parts: list[pd.DataFrame] = [] - for title, df in blocks: - df2 = df.copy() - # Put the section label as the first column for readability. - df2.insert(0, "Section", title) - combined_parts.append(df2) - - combined = pd.concat(combined_parts, axis=0, ignore_index=True, sort=False) - combined.to_excel(writer, sheet_name=sheet, index=False) - - -def _safe_filename(s: str) -> str: - # Fast path without the third-party `regex` module. - s = " ".join(str(s).strip().split()) - allowed = [] - for ch in s: - if ch.isalnum() or ch in "._-": - allowed.append(ch) - else: - allowed.append("_") - out = "".join(allowed) - return out[:180] if len(out) > 180 else out - - -# ----------------------------- -# vLLM environment export helper -# ----------------------------- -def _parse_vllm_env_txt(env_path: Path) -> pd.DataFrame: - """Parse vllm_env.txt into a flat table (Section, Key, Value). - - Supports: - - section headers as standalone lines (no ':' or '=') - - key-value lines like 'OS: Ubuntu ...' - - env var lines like 'HF_HOME=/data/hf' - """ - lines = env_path.read_text(encoding="utf-8", errors="replace").splitlines() - section = "General" - rows: list[dict] = [] - - def set_section(s: str): - nonlocal section - s = (s or "").strip() - if s: - section = s - - for raw in lines: - stripped = raw.strip() - if not stripped: - continue - # divider lines like ===== - if set(stripped) <= {"="}: - continue - - # section header heuristic: short standalone line - if ":" not in stripped and "=" not in stripped and len(stripped) <= 64: - if stripped.lower().startswith("collecting environment information"): - continue - set_section(stripped) - continue - - # env var style: KEY=VALUE (and not a URL with :) - if "=" in stripped and ":" not in stripped: - k, v = stripped.split("=", 1) - k = k.strip() - v = v.strip() - if k: - rows.append({"Section": section, "Key": k, "Value": v}) - continue - - # key: value - if ":" in stripped: - k, v = stripped.split(":", 1) - k = k.strip() - v = v.strip() - if k: - rows.append({"Section": section, "Key": k, "Value": v}) - continue - - return pd.DataFrame(rows, columns=["Section", "Key", "Value"]) - - -def _load_env_df_for_inputs(args, files: list[str]) -> pd.DataFrame | None: - """Load vllm_env.txt next to the *original* input JSON file. - - Note: when only one -f is provided, the script may split JSON into ./splits/..., - but vllm_env.txt typically lives next to the original benchmark_results.json. - """ - base_dir: Path | None = None - if getattr(args, "file", None): - base_dir = Path(args.file[0]).resolve().parent - elif files: - base_dir = Path(files[0]).resolve().parent - if base_dir is None: - return None - - env_path = base_dir / "vllm_env.txt" - if not env_path.exists(): - return None - df = _parse_vllm_env_txt(env_path) - return df - - -# ----------------------------- -# Valid max concurrency summary helpers -# ----------------------------- -def _config_value_columns(df: pd.DataFrame, conc_col: str) -> list[str]: - key_cols = [ - c - for c in ["Model", "Dataset Name", "Input Len", "Output Len"] - if c in df.columns - ] - exclude = set(key_cols + [conc_col, "qps", "QPS"]) - - cols: list[str] = [] - for c in df.columns: - if c in exclude: - continue - lc = str(c).lower() - if lc.startswith("ratio"): - continue - if lc.endswith("_name") or lc == "test name" or lc == "test_name": - continue - if pd.api.types.is_numeric_dtype(df[c]): - cols.append(c) - return cols - - -def _max_concurrency_ok( - df: pd.DataFrame, - conc_col: str, - cfg_col: str, - threshold: float, - slack_pct: float = 0.0, -): - if df is None or conc_col not in df.columns or cfg_col not in df.columns: - return pd.NA - - d = df[[conc_col, cfg_col]].copy() - d[conc_col] = pd.to_numeric(d[conc_col], errors="coerce") - d[cfg_col] = pd.to_numeric(d[cfg_col], errors="coerce") - d = d.dropna(subset=[conc_col, cfg_col]) - - if d.empty: - return pd.NA - - # Accept values up to (1 + slack_pct%) above the SLA. - try: - slack_pct = float(slack_pct or 0.0) - except Exception: - slack_pct = 0.0 - effective_limit = float(threshold) * (1.0 + slack_pct / 100.0) - - ok = d[d[cfg_col] <= effective_limit] - if ok.empty: - return pd.NA - - return ok[conc_col].max() - - -def _value_at_concurrency(df: pd.DataFrame, conc_col: str, cfg_col: str, conc_value): - if ( - df is None - or conc_col not in df.columns - or cfg_col not in df.columns - or pd.isna(conc_value) - ): - return pd.NA - - d = df[[conc_col, cfg_col]].copy() - d[conc_col] = pd.to_numeric(d[conc_col], errors="coerce") - d[cfg_col] = pd.to_numeric(d[cfg_col], errors="coerce") - - conc_value = pd.to_numeric(conc_value, errors="coerce") - if pd.isna(conc_value): - return pd.NA - - hit = d[d[conc_col] == conc_value] - if hit.empty: - return pd.NA - return hit[cfg_col].iloc[0] - - -def build_valid_max_concurrency_summary_html( - tput_group_df: pd.DataFrame | None, - ttft_group_df: pd.DataFrame | None, - tpot_group_df: pd.DataFrame | None, - conc_col: str, - args, -) -> str: - if ttft_group_df is None and tpot_group_df is None: - return "" - - ttft_cols = ( - _config_value_columns(ttft_group_df, conc_col) - if ttft_group_df is not None - else [] - ) - tpot_cols = ( - _config_value_columns(tpot_group_df, conc_col) - if tpot_group_df is not None - else [] - ) - tput_cols = ( - _config_value_columns(tput_group_df, conc_col) - if tput_group_df is not None - else [] - ) - - if ttft_group_df is not None and tpot_group_df is not None: - cfg_cols = [c for c in ttft_cols if c in tpot_cols] - if tput_group_df is not None: - cfg_cols = [c for c in cfg_cols if c in tput_cols] or cfg_cols - else: - cfg_cols = ttft_cols or tpot_cols - - if not cfg_cols: - cfg_cols = sorted(set(ttft_cols) | set(tpot_cols) | set(tput_cols), key=str) - - # Display SLA ranges in the table header (SLA .. SLA*(1+slack)) - ttft_hi = args.ttft_max_ms * (1.0 + args.ttft_slack_pct / 100.0) - tpot_hi = args.tpot_max_ms * (1.0 + args.tpot_slack_pct / 100.0) - ttft_range = f"{args.ttft_max_ms:g}–{ttft_hi:g} ms (+{args.ttft_slack_pct:g}%)" - tpot_range = f"{args.tpot_max_ms:g}–{tpot_hi:g} ms (+{args.tpot_slack_pct:g}%)" - - rows = [] - for cfg in cfg_cols: - ttft_max = ( - _max_concurrency_ok( - ttft_group_df, conc_col, cfg, args.ttft_max_ms, args.ttft_slack_pct - ) - if ttft_group_df is not None - else pd.NA - ) - tpot_max = ( - _max_concurrency_ok( - tpot_group_df, conc_col, cfg, args.tpot_max_ms, args.tpot_slack_pct - ) - if tpot_group_df is not None - else pd.NA - ) - both = ( - pd.NA - if (pd.isna(ttft_max) or pd.isna(tpot_max)) - else min(ttft_max, tpot_max) - ) - - tput_at_both = ( - _value_at_concurrency(tput_group_df, conc_col, cfg, both) - if tput_group_df is not None - else pd.NA - ) - ttft_at_both = ( - _value_at_concurrency(ttft_group_df, conc_col, cfg, both) - if ttft_group_df is not None - else pd.NA - ) - tpot_at_both = ( - _value_at_concurrency(tpot_group_df, conc_col, cfg, both) - if tpot_group_df is not None - else pd.NA - ) - - rows.append( - { - "Configuration": cfg, - f"Max {conc_col} (TTFT ≤ {ttft_range})": ttft_max, - f"Max {conc_col} (TPOT ≤ {tpot_range})": tpot_max, - f"Max {conc_col} (Both)": both, - "Output Tput @ Both (tok/s)": tput_at_both, - "TTFT @ Both (ms)": ttft_at_both, - "TPOT @ Both (ms)": tpot_at_both, - } - ) - - summary_df = pd.DataFrame(rows) - - for c in summary_df.columns: - if c == "Configuration": - continue - summary_df[c] = pd.to_numeric(summary_df[c], errors="coerce") - - both_col = f"Max {conc_col} (Both)" - - formatters = {} - for c in summary_df.columns: - if c == "Configuration": - continue - formatters[c] = lambda v: "" if pd.isna(v) else f"{float(v):.2f}" - - styler = summary_df.style.format(formatters) - - def _green(v): - return "background-color:#e6ffe6;font-weight:bold;" if pd.notna(v) else "" - - if both_col in summary_df.columns: - styler = styler.map(_green, subset=[both_col]) - - title = ( - '
' - "Valid Max Concurrency Summary" - "
\n" - ) - return title + styler.to_html(table_attributes='border="1" class="dataframe"') - - -def build_valid_max_concurrency_summary_df( - tput_group_df: pd.DataFrame | None, - ttft_group_df: pd.DataFrame | None, - tpot_group_df: pd.DataFrame | None, - conc_col: str, - args, -) -> pd.DataFrame | None: - if ttft_group_df is None and tpot_group_df is None: - return None - - ttft_cols = ( - _config_value_columns(ttft_group_df, conc_col) - if ttft_group_df is not None - else [] - ) - tpot_cols = ( - _config_value_columns(tpot_group_df, conc_col) - if tpot_group_df is not None - else [] - ) - tput_cols = ( - _config_value_columns(tput_group_df, conc_col) - if tput_group_df is not None - else [] - ) - - if ttft_group_df is not None and tpot_group_df is not None: - cfg_cols = [c for c in ttft_cols if c in tpot_cols] - if tput_group_df is not None: - cfg_cols = [c for c in cfg_cols if c in tput_cols] or cfg_cols - else: - cfg_cols = ttft_cols or tpot_cols - - if not cfg_cols: - cfg_cols = sorted(set(ttft_cols) | set(tpot_cols) | set(tput_cols), key=str) - - ttft_hi = args.ttft_max_ms * (1.0 + args.ttft_slack_pct / 100.0) - tpot_hi = args.tpot_max_ms * (1.0 + args.tpot_slack_pct / 100.0) - ttft_range = f"{args.ttft_max_ms:g}–{ttft_hi:g} ms (+{args.ttft_slack_pct:g}%)" - tpot_range = f"{args.tpot_max_ms:g}–{tpot_hi:g} ms (+{args.tpot_slack_pct:g}%)" - - rows = [] - for cfg in cfg_cols: - ttft_max = ( - _max_concurrency_ok( - ttft_group_df, conc_col, cfg, args.ttft_max_ms, args.ttft_slack_pct - ) - if ttft_group_df is not None - else pd.NA - ) - tpot_max = ( - _max_concurrency_ok( - tpot_group_df, conc_col, cfg, args.tpot_max_ms, args.tpot_slack_pct - ) - if tpot_group_df is not None - else pd.NA - ) - both = ( - pd.NA - if (pd.isna(ttft_max) or pd.isna(tpot_max)) - else min(ttft_max, tpot_max) - ) - - tput_at_both = ( - _value_at_concurrency(tput_group_df, conc_col, cfg, both) - if tput_group_df is not None - else pd.NA - ) - ttft_at_both = ( - _value_at_concurrency(ttft_group_df, conc_col, cfg, both) - if ttft_group_df is not None - else pd.NA - ) - tpot_at_both = ( - _value_at_concurrency(tpot_group_df, conc_col, cfg, both) - if tpot_group_df is not None - else pd.NA - ) - - rows.append( - { - "Configuration": cfg, - f"Max {conc_col} (TTFT ≤ {ttft_range})": ttft_max, - f"Max {conc_col} (TPOT ≤ {tpot_range})": tpot_max, - f"Max {conc_col} (Both)": both, - "Output Tput @ Both (tok/s)": tput_at_both, - "TTFT @ Both (ms)": ttft_at_both, - "TPOT @ Both (ms)": tpot_at_both, - } - ) - - df = pd.DataFrame(rows) - for c in df.columns: - if c != "Configuration": - df[c] = pd.to_numeric(df[c], errors="coerce") - return df - - -# ----------------------------- -# Plot helper -# ----------------------------- -def _add_limit_line(fig, y_value: float, label: str): - fig.add_hline( - y=y_value, - line_dash="dash", - line_color="red" if "ttft" in label.lower() else "blue", - annotation_text=f"{label}: {y_value} ms", - annotation_position="top left", - ) - if plotly_found: - import plotly.graph_objects as go - - fig.add_trace( - go.Scatter( - x=[None], - y=[None], - mode="lines", - line=dict( - dash="dash", - color="red" if "ttft" in label.lower() else "blue", - ), - name=label, - ) - ) - - -# ----------------------------- -# Refactored main + group-first report -# ----------------------------- -@dataclass(frozen=True) -class MetricPlan: - data_cols: list[str] - drop_column: str - - -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser() - parser.add_argument( - "-f", "--file", action="append", type=str, help="input file name" - ) - parser.add_argument( - "--debug", action="store_true", help="show all information for debugging" - ) - parser.add_argument( - "--plot", - action=argparse.BooleanOptionalAction, - default=True, - help="plot perf diagrams or not --no-plot --plot", - ) - parser.add_argument( - "-x", - "--xaxis", - type=str, - default="# of max concurrency.", - help="column name to use as X Axis in comparison graph", - ) - parser.add_argument( - "-l", - "--latency", - type=str, - default="p99", - help="take median|p99 for latency like TTFT/TPOT", - ) - parser.add_argument( - "--ttft-max-ms", - type=float, - default=3000.0, - help="Reference limit for TTFT plots (ms)", - ) - parser.add_argument( - "--tpot-max-ms", - type=float, - default=100.0, - help="Reference limit for TPOT plots (ms)", - ) - - # ---- SLA tolerance (slack) options ---- - parser.add_argument( - "--ttft-slack-pct", - type=float, - default=5.0, - help="Allowed percentage above TTFT SLA (default: 5).", - ) - parser.add_argument( - "--tpot-slack-pct", - type=float, - default=5.0, - help="Allowed percentage above TPOT SLA (default: 5).", - ) - - # ---- export options ---- - parser.add_argument( - "--excel-out", - type=str, - default="perf_comparison.xlsx", - help="Write one sheet per (Model, Dataset, Input Len, Output Len).", - ) - parser.add_argument( - "--csv-out-dir", - type=str, - default="", - help="If set, write per-group per-metric CSVs into this directory.", - ) - - return parser - - -def choose_metrics(latency: str) -> MetricPlan: - latency = (latency or "").lower() - drop_column = "P99" - - if "median" in latency: - return MetricPlan( - data_cols=["Output Tput (tok/s)", "Median TTFT (ms)", "Median"], - drop_column=drop_column, - ) - - return MetricPlan( - data_cols=["Output Tput (tok/s)", "P99 TTFT (ms)", "P99"], - drop_column=drop_column, - ) - - -def prepare_input_files(args, info_cols: list[str]) -> tuple[list[str], list[str]]: - if not args.file: - raise ValueError("No input files provided. Use -f/--file.") - - if len(args.file) == 1: - files = split_json_by_tp_pp(args.file[0], output_root="splits") - info_cols = [c for c in info_cols if c not in ("TP Size", "PP Size")] - else: - files = args.file - - return files, info_cols - - -def get_y_axis_col(info_cols: list[str], xaxis: str) -> str: - y_axis_index = info_cols.index(xaxis) if xaxis in info_cols else 6 - return info_cols[y_axis_index] - - -def get_group_cols(output_df: pd.DataFrame, info_cols: list[str]) -> list[str]: - filtered_info_cols = info_cols[:4] - group_cols = [c for c in filtered_info_cols if c in output_df.columns] - if not group_cols: - raise ValueError( - f"No valid group-by columns. Expected subset: {filtered_info_cols}, " - f"but DataFrame has: {list(output_df.columns)}" - ) - return group_cols - - -def normalize_group_key(name): - return name if isinstance(name, tuple) else (name,) - - -def group_filename(name, prefix: str = "perf_comparison_") -> str: - name_vals = normalize_group_key(name) - safe = ",".join(map(str, name_vals)).replace(",", "_").replace("/", "-") - return f"{prefix}{safe}.html" - - -def build_group_suffix(group_cols: list[str], name) -> str: - name_vals = normalize_group_key(name) - return " , ".join(f"{col} : [ {val} ] " for col, val in zip(group_cols, name_vals)) - - -def render_metric_table_html( - display_group: pd.DataFrame, - metric_label: str, - group_suffix: str, - args, -) -> str: - title = ( - f'
' - f"{_html.escape(metric_label)}" - f" — {_html.escape(group_suffix)}" - f"
\n" - ) - - metric_name = metric_label.lower() - if "ttft" in metric_name: - styler = _highlight_threshold( - display_group, args.ttft_max_ms, args.ttft_slack_pct - ) - elif ("tpot" in metric_name) or ("median" in metric_name) or ("p99" in metric_name): - styler = _highlight_threshold( - display_group, args.tpot_max_ms, args.tpot_slack_pct - ) - else: - styler = display_group.style - - styler = _apply_two_decimals(styler) - styler = highlight_ratio_columns(styler) - - return title + styler.to_html(table_attributes='border="1" class="dataframe"') - - -def maybe_write_plot( - main_fh, - sub_fh, - group_df: pd.DataFrame, - raw_data_cols: list[str], - metric_label: str, - y_axis_col: str, - args, -): - if not (args.plot and plotly_found): - return - - import plotly.express as px - - df = group_df[raw_data_cols].sort_values(by=y_axis_col) - df_melted = df.melt( - id_vars=y_axis_col, - var_name="Configuration", - value_name=metric_label, - ) - - fig = px.line( - df_melted, - x=y_axis_col, - y=metric_label, - color="Configuration", - title=f"{metric_label} vs {y_axis_col}", - markers=True, - ) - - fig.update_traces(hovertemplate="%{y:.2f}") - fig.update_yaxes(tickformat=".2f") - - metric_name = metric_label.lower() - if "ttft" in metric_name: - _add_limit_line(fig, args.ttft_max_ms, "TTFT limit") - elif ("tpot" in metric_name) or ("median" in metric_name) or ("p99" in metric_name): - _add_limit_line(fig, args.tpot_max_ms, "TPOT limit") - - html = fig.to_html(full_html=True, include_plotlyjs="cdn") - main_fh.write(html) - sub_fh.write(html) - - -def build_group_keys( - df: pd.DataFrame, group_cols: list[str], sort_cols: list[str] | None = None -): - if sort_cols: - df = df.sort_values(by=sort_cols) - gb = df.groupby(group_cols, dropna=False) - return [k for k, _ in gb] - - -def write_report_group_first( - files: list[str], info_cols: list[str], plan: MetricPlan, args -): - name_column = "Test name" - y_axis_col = get_y_axis_col(info_cols, args.xaxis) - - print("comparing : " + ", ".join(files)) - - metric_cache: dict[str, tuple[pd.DataFrame, list[str]]] = {} - group_cols_canonical: list[str] | None = None - - for metric_label in plan.data_cols: - output_df, raw_data_cols = compare_data_columns( - files, - name_column, - metric_label, - info_cols, - plan.drop_column, - debug=args.debug, - ) - - raw_data_cols = list(raw_data_cols) - raw_data_cols.insert(0, y_axis_col) - - group_cols = get_group_cols(output_df, info_cols) - if group_cols_canonical is None: - group_cols_canonical = group_cols - else: - group_cols_canonical = [c for c in group_cols_canonical if c in group_cols] - - metric_cache[metric_label] = ( - output_df.sort_values(by=args.xaxis), - raw_data_cols, - ) - - if not group_cols_canonical: - raise ValueError("No canonical group columns found across metrics.") - - first_metric = plan.data_cols[0] - first_df_sorted, _ = metric_cache[first_metric] - group_keys = build_group_keys( - first_df_sorted, group_cols_canonical, sort_cols=[args.xaxis] - ) - - metric_groupbys = { - metric_label: df.groupby(group_cols_canonical, dropna=False) - for metric_label, (df, _) in metric_cache.items() - } - - csv_dir = Path(args.csv_out_dir) if args.csv_out_dir else None - if csv_dir: - csv_dir.mkdir(parents=True, exist_ok=True) - - excel_path = args.excel_out or "perf_comparison.xlsx" - disable_excel = os.getenv("VLLM_COMPARE_DISABLE_EXCEL", "0") == "1" - - # Prefer xlsxwriter for speed; fallback to openpyxl if unavailable. - excel_engine = ( - os.getenv("VLLM_COMPARE_EXCEL_ENGINE", "xlsxwriter").strip() or "xlsxwriter" - ) - if excel_engine == "xlsxwriter" and util.find_spec("xlsxwriter") is None: - excel_engine = "openpyxl" - - excel_engine_kwargs = {} - if excel_engine == "xlsxwriter": - # Reduce memory pressure & usually faster writes. - excel_engine_kwargs = {"options": {"constant_memory": True}} - - xw_ctx = ( - nullcontext(None) - if disable_excel - else pd.ExcelWriter( - excel_path, engine=excel_engine, engine_kwargs=excel_engine_kwargs - ) - ) - with xw_ctx as xw: - used_sheets: set[str] = set() - # ---- Environment sheet (first) ---- - env_sheet = _sanitize_sheet_name("Environment") - env_df = _load_env_df_for_inputs(args, files) - if xw is not None: - if env_df is None or env_df.empty: - pd.DataFrame( - [ - { - "Section": "Environment", - "Key": "vllm_env.txt", - "Value": "NOT FOUND (or empty)", - } - ] - ).to_excel(xw, sheet_name=env_sheet, index=False) - else: - env_df.to_excel(xw, sheet_name=env_sheet, index=False) - used_sheets.add(env_sheet) - with open("perf_comparison.html", "w", encoding="utf-8") as main_fh: - main_fh.write('\n') - for gkey in group_keys: - gkey_tuple = normalize_group_key(gkey) - suffix = build_group_suffix(group_cols_canonical, gkey_tuple) - sub_path = group_filename(gkey_tuple) - group_header = ( - '
' - f"{_html.escape(suffix)}" - "
\n" - ) - - main_fh.write(group_header) - - do_excel = xw is not None - sheet = _group_to_sheet_base(group_cols_canonical, gkey_tuple) - sheet_base = sheet - if do_excel: - dedup_i = 1 - while sheet in used_sheets: - dedup_i += 1 - suffix = f"_{dedup_i}" - # Ensure uniqueness even when sheet names are truncated. - base = str(sheet_base) - keep = max(1, 31 - len(suffix)) - sheet = _sanitize_sheet_name(base[:keep] + suffix) - used_sheets.add(sheet) - - excel_blocks: list[tuple[str, pd.DataFrame]] = [] - - with open(sub_path, "w", encoding="utf-8") as sub_fh: - sub_fh.write('\n') - sub_fh.write(group_header) - tput_group_df = None - ttft_group_df = None - tpot_group_df = None - conc_col = args.xaxis - - for metric_label in plan.data_cols: - gb = metric_groupbys[metric_label] - df_sorted, raw_data_cols = metric_cache[metric_label] - - try: - group_df = gb.get_group(gkey) - except KeyError: - missing = ( - '
' - f"{_html.escape(metric_label)} — missing for this group" - "
\n" - ) - main_fh.write(missing) - sub_fh.write(missing) - continue - - if conc_col not in group_df.columns: - conc_col = _find_concurrency_col(group_df) - - mn = metric_label.lower().strip() - if "tok/s" in mn: - tput_group_df = group_df - elif "ttft" in mn: - ttft_group_df = group_df - elif mn in ("p99", "median") or "tpot" in mn: - tpot_group_df = group_df - - display_group = group_df.drop( - columns=group_cols_canonical, errors="ignore" - ) - - html = render_metric_table_html( - display_group, metric_label, suffix, args - ) - main_fh.write(html) - sub_fh.write(html) - - maybe_write_plot( - main_fh, - sub_fh, - group_df=group_df, - raw_data_cols=raw_data_cols, - metric_label=metric_label, - y_axis_col=y_axis_col, - args=args, - ) - - excel_blocks.append( - (metric_label, group_df.reset_index(drop=True)) - ) - if csv_dir: - fn = _safe_filename( - f"{sheet}__{metric_label}".replace(" ", "_").replace( - "/", "_" - ) - ) - group_df.to_csv(csv_dir / f"{fn}.csv", index=False) - - summary_html = build_valid_max_concurrency_summary_html( - tput_group_df=tput_group_df, - ttft_group_df=ttft_group_df, - tpot_group_df=tpot_group_df, - conc_col=conc_col, - args=args, - ) - if summary_html: - main_fh.write(summary_html) - sub_fh.write(summary_html) - - summary_df = build_valid_max_concurrency_summary_df( - tput_group_df=tput_group_df, - ttft_group_df=ttft_group_df, - tpot_group_df=tpot_group_df, - conc_col=conc_col, - args=args, - ) - if summary_df is not None: - excel_blocks.append( - ("Valid Max Concurrency Summary", summary_df) - ) - if csv_dir: - fn = _safe_filename( - f"{sheet}__Valid_Max_Concurrency_Summary" - ) - summary_df.to_csv(csv_dir / f"{fn}.csv", index=False) - - if do_excel: - _write_tables_to_excel_sheet(xw, sheet, excel_blocks) - - if disable_excel: - print("Skipped Excel generation (VLLM_COMPARE_DISABLE_EXCEL=1).") - else: - print(f"Wrote Excel: {excel_path}") - if csv_dir: - print(f"Wrote CSVs under: {csv_dir}") - - -def main(): - args = build_parser().parse_args() - info_cols = list(DEFAULT_INFO_COLS) - plan = choose_metrics(args.latency) - files, info_cols = prepare_input_files(args, info_cols) - write_report_group_first(files, info_cols, plan, args) - - -if __name__ == "__main__": - main() diff --git a/.buildkite/performance-benchmarks/scripts/convert-results-json-to-markdown.py b/.buildkite/performance-benchmarks/scripts/convert-results-json-to-markdown.py deleted file mode 100644 index deb14a65e4..0000000000 --- a/.buildkite/performance-benchmarks/scripts/convert-results-json-to-markdown.py +++ /dev/null @@ -1,413 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import argparse -import json -import os -import shlex -from importlib import util -from pathlib import Path -from typing import Any - -import pandas as pd -import psutil -import regex as re -from tabulate import tabulate - -# latency results and the keys that will be printed into markdown -latency_results = [] -latency_column_mapping = { - "test_name": "Test name", - "gpu_type": "GPU", - "avg_latency": "Mean latency (ms)", - # "P10": "P10 (s)", - # "P25": "P25 (s)", - "P50": "Median latency (ms)", - # "P75": "P75 (s)", - # "P90": "P90 (s)", - "P99": "P99 latency (ms)", -} - -# throughput tests and the keys that will be printed into markdown -throughput_results = [] -throughput_results_column_mapping = { - "test_name": "Test name", - "gpu_type": "GPU", - "num_requests": "# of req.", - "total_num_tokens": "Total # of tokens", - "elapsed_time": "Elapsed time (s)", - "requests_per_second": "Tput (req/s)", - "tokens_per_second": "Tput (tok/s)", -} - -# serving results and the keys that will be printed into markdown -serving_results = [] -serving_column_mapping = { - "test_name": "Test name", - "model_id": "Model", - "dataset_name": "Dataset Name", - "input_len": "Input Len", - "output_len": "Output Len", - "tp_size": "TP Size", - "pp_size": "PP Size", - "dtype": "dtype", - "gpu_type": "GPU", - "completed": "# of req.", - "qps": "qps", - "max_concurrency": "# of max concurrency.", - "request_throughput": "Tput (req/s)", - "total_token_throughput": "Total Token Tput (tok/s)", - "output_throughput": "Output Tput (tok/s)", - # "total_input_tokens": "Total input tokens", - # "total_output_tokens": "Total output tokens", - "mean_ttft_ms": "Mean TTFT (ms)", - "median_ttft_ms": "Median TTFT (ms)", - "p99_ttft_ms": "P99 TTFT (ms)", - "std_ttft_ms": "STD TTFT (ms)", - "mean_tpot_ms": "Mean TPOT (ms)", - "median_tpot_ms": "Median", - "p99_tpot_ms": "P99", - "std_tpot_ms": "STD TPOT (ms)", - "mean_itl_ms": "Mean ITL (ms)", - "median_itl_ms": "Median ITL (ms)", - "p99_itl_ms": "P99 ITL (ms)", -} - - -def read_markdown(file): - if os.path.exists(file): - with open(file) as f: - return f.read() + "\n" - else: - return f"{file} not found.\n" - - -def results_to_json(latency, throughput, serving): - return json.dumps( - { - "latency": latency.to_dict(), - "throughput": throughput.to_dict(), - "serving": serving.to_dict(), - } - ) - - -def get_size_with_unit(bytes, suffix="B"): - """Scale bytes to its proper format - e.g: - 1253656 => '1.20MB' - 1253656678 => '1.17GB' - """ - factor = 1024 - for unit in ["", "K", "M", "G", "T", "P"]: - if bytes < factor: - return f"{bytes:.2f}{unit}{suffix}" - bytes /= factor - - -def _coerce(val: str) -> Any: - """Best-effort type coercion from string to Python types.""" - low = val.lower() - if low == "null": - return None - if low == "true": - return True - if low == "false": - return False - # integers - if re.fullmatch(r"[+-]?\d+", val): - try: - return int(val) - except ValueError: - pass - # floats (keep 'inf'/'-inf'/'nan' as strings) - if re.fullmatch(r"[+-]?\d*\.\d+", val): - try: - return float(val) - except ValueError: - pass - return val - - -def parse_client_command(cmd: str) -> dict[str, Any]: - """Parse the client_command shell string into {executable, script, args}.""" - toks = shlex.split(cmd) - if len(toks) < 2: - raise ValueError("client_command must include an executable and a script") - executable, script = toks[0], toks[1] - args: dict[str, Any] = {} - - i = 2 - while i < len(toks): - t = toks[i] - if t.startswith("--"): - # --key=value or --key (value) or boolean flag - if "=" in t: - key, val = t.split("=", 1) - if key == "--metadata": - md = {} - if val: - if "=" in val: - k, v = val.split("=", 1) - md[k] = _coerce(v) - else: - md[val] = True - args[key] = md - else: - args[key] = _coerce(val) - i += 1 - continue - - key = t - - # Special: consume metadata k=v pairs until next --flag - if key == "--metadata": - i += 1 - md = {} - while i < len(toks) and not toks[i].startswith("--"): - pair = toks[i] - if "=" in pair: - k, v = pair.split("=", 1) - md[k] = _coerce(v) - else: - md[pair] = True - i += 1 - args[key] = md - continue - - # Standard: check if next token is a value (not a flag) - if i + 1 < len(toks) and not toks[i + 1].startswith("--"): - args[key] = _coerce(toks[i + 1]) - i += 2 - else: - # lone flag -> True - args[key] = True - i += 1 - else: - # unexpected positional; skip - i += 1 - - return {"executable": executable, "script": script, "args": args} - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "-r", - "--result", - type=str, - default="results", - help="Folder name for benchmark output results.", - ) - args = parser.parse_args() - results_folder = Path(args.result) - if not results_folder.exists(): - raise FileNotFoundError(f"results folder does not exist: {results_folder}") - # collect results - for test_file in results_folder.glob("*.json"): - with open(test_file) as f: - raw_result = json.loads(f.read()) - - if "serving" in str(test_file): - # this result is generated via `vllm bench serve` command - # attach the benchmarking command to raw_result - try: - with open(test_file.with_suffix(".commands")) as f: - command = json.loads(f.read()) - except OSError as e: - print(e) - continue - # Parse Server Command Arg - out: dict[str, Any] = { - "server_command": parse_client_command(command["server_command"]) - } - parse_args = [ - "--tensor-parallel-size", - "--pipeline-parallel-size", - "--dtype", - ] - col_mapping = ["tp_size", "pp_size", "dtype"] - for index, arg in enumerate(parse_args): - if arg in out["server_command"]["args"]: - raw_result.update( - {col_mapping[index]: out["server_command"]["args"][arg]} - ) - - # Parse Client Command Arg - out: dict[str, Any] = { - "client_command": parse_client_command(command["client_command"]) - } - parse_args = [ - "--dataset-name", - "--random-input-len", - "--random-output-len", - "--request-rate", - ] - col_mapping = ["dataset_name", "input_len", "output_len", "qps"] - - for index, arg in enumerate(parse_args): - if arg in out["client_command"]["args"]: - raw_result.update( - {col_mapping[index]: out["client_command"]["args"][arg]} - ) - # Add Server, Client command - raw_result.update(command) - - # update the test name of this result - raw_result.update({"test_name": test_file.stem}) - # add the result to raw_result - serving_results.append(raw_result) - continue - - elif "latency" in f.name: - # this result is generated via `vllm bench latency` command - - # attach the benchmarking command to raw_result - try: - with open(test_file.with_suffix(".commands")) as f: - command = json.loads(f.read()) - except OSError as e: - print(e) - continue - - raw_result.update(command) - - # update the test name of this result - raw_result.update({"test_name": test_file.stem}) - - # get different percentiles - for perc in [10, 25, 50, 75, 90, 99]: - # Multiply 1000 to convert the time unit from s to ms - raw_result.update( - {f"P{perc}": 1000 * raw_result["percentiles"][str(perc)]} - ) - raw_result["avg_latency"] = raw_result["avg_latency"] * 1000 - - # add the result to raw_result - latency_results.append(raw_result) - continue - - elif "throughput" in f.name: - # this result is generated via `vllm bench throughput` command - - # attach the benchmarking command to raw_result - try: - with open(test_file.with_suffix(".commands")) as f: - command = json.loads(f.read()) - except OSError as e: - print(e) - continue - - raw_result.update(command) - - # update the test name of this result - raw_result.update({"test_name": test_file.stem}) - - # add the result to raw_result - throughput_results.append(raw_result) - continue - - print(f"Skipping {test_file}") - - latency_results = pd.DataFrame.from_dict(latency_results) - serving_results = pd.DataFrame.from_dict(serving_results) - throughput_results = pd.DataFrame.from_dict(throughput_results) - - svmem = psutil.virtual_memory() - platform_data = { - "Physical cores": [psutil.cpu_count(logical=False)], - "Total cores": [psutil.cpu_count(logical=True)], - "Total Memory": [get_size_with_unit(svmem.total)], - } - - if util.find_spec("numa") is not None: - from numa import info - - platform_data["Total NUMA nodes"] = [info.get_num_configured_nodes()] - - if util.find_spec("cpuinfo") is not None: - from cpuinfo import get_cpu_info - - platform_data["CPU Brand"] = [get_cpu_info()["brand_raw"]] - - platform_results = pd.DataFrame.from_dict( - platform_data, orient="index", columns=["Platform Info"] - ) - - raw_results_json = results_to_json( - latency_results, throughput_results, serving_results - ) - - # remapping the key, for visualization purpose - if not latency_results.empty: - latency_results = latency_results[list(latency_column_mapping.keys())].rename( - columns=latency_column_mapping - ) - if not serving_results.empty: - valid_columns = [ - col for col in serving_column_mapping if col in serving_results.columns - ] - serving_results = serving_results[valid_columns].rename( - columns=serving_column_mapping - ) - if not throughput_results.empty: - throughput_results = throughput_results[ - list(throughput_results_column_mapping.keys()) - ].rename(columns=throughput_results_column_mapping) - - processed_results_json = results_to_json( - latency_results, throughput_results, serving_results - ) - - for df in [latency_results, serving_results, throughput_results]: - if df.empty: - continue - - # Sort all dataframes by their respective "Test name" columns - df.sort_values(by="Test name", inplace=True) - - # The GPUs sometimes come in format of "GPUTYPE\nGPUTYPE\n...", - # we want to turn it into "8xGPUTYPE" - df["GPU"] = df["GPU"].apply( - lambda x: "{}x{}".format(len(x.split("\n")), x.split("\n")[0]) - ) - - # get markdown tables - latency_md_table = tabulate( - latency_results, headers="keys", tablefmt="pipe", showindex=False - ) - serving_md_table = tabulate( - serving_results, headers="keys", tablefmt="pipe", showindex=False - ) - throughput_md_table = tabulate( - throughput_results, headers="keys", tablefmt="pipe", showindex=False - ) - platform_md_table = tabulate( - platform_results, headers="keys", tablefmt="pipe", showindex=True - ) - - # document the result - md_file = "benchmark_results.md" - json_file = "benchmark_results.json" - with open(results_folder / md_file, "w") as f: - results = read_markdown( - "../.buildkite/performance-benchmarks/" - "performance-benchmarks-descriptions.md" - ) - results = results.format( - latency_tests_markdown_table=latency_md_table, - throughput_tests_markdown_table=throughput_md_table, - serving_tests_markdown_table=serving_md_table, - platform_markdown_table=platform_md_table, - benchmarking_results_in_json_string=processed_results_json, - ) - f.write(results) - - # document benchmarking results in json - with open(results_folder / json_file, "w") as f: - results = ( - latency_results.to_dict(orient="records") - + throughput_results.to_dict(orient="records") - + serving_results.to_dict(orient="records") - ) - f.write(json.dumps(results)) diff --git a/.buildkite/performance-benchmarks/scripts/launch-server.sh b/.buildkite/performance-benchmarks/scripts/launch-server.sh deleted file mode 100644 index ebacdcbd68..0000000000 --- a/.buildkite/performance-benchmarks/scripts/launch-server.sh +++ /dev/null @@ -1,224 +0,0 @@ -#!/bin/bash - -# Currently FP8 benchmark is NOT enabled. - -set -x -server_params=$1 -common_params=$2 - -json2args() { - # transforms the JSON string to command line args, and '_' is replaced to '-' - # example: - # input: { "model": "meta-llama/Llama-2-7b-chat-hf", "tensor_parallel_size": 1 } - # output: --model meta-llama/Llama-2-7b-chat-hf --tensor-parallel-size 1 - local json_string=$1 - local args=$( - echo "$json_string" | jq -r ' - to_entries | - map("--" + (.key | gsub("_"; "-")) + " " + (.value | tostring)) | - join(" ") - ' - ) - echo "$args" -} - -launch_trt_server() { - - model_path=$(echo "$common_params" | jq -r '.model') - model_name="${model_path#*/}" - model_type=$(echo "$server_params" | jq -r '.model_type') - model_dtype=$(echo "$server_params" | jq -r '.model_dtype') - model_tp_size=$(echo "$common_params" | jq -r '.tp') - max_batch_size=$(echo "$server_params" | jq -r '.max_batch_size') - max_input_len=$(echo "$server_params" | jq -r '.max_input_len') - max_seq_len=$(echo "$server_params" | jq -r '.max_seq_len') - max_num_tokens=$(echo "$server_params" | jq -r '.max_num_tokens') - trt_llm_version=$(echo "$server_params" | jq -r '.trt_llm_version') - - # create model caching directory - cd ~ - rm -rf models - mkdir -p models - cd models - models_dir=$(pwd) - trt_model_path=${models_dir}/${model_name}-trt-ckpt - trt_engine_path=${models_dir}/${model_name}-trt-engine - - # clone tensorrt backend - cd / - rm -rf tensorrtllm_backend - git clone https://github.com/triton-inference-server/tensorrtllm_backend.git - git lfs install - cd tensorrtllm_backend - git checkout "$trt_llm_version" - git submodule update --init --recursive - - # build trtllm engine - cd /tensorrtllm_backend - cd "./tensorrt_llm/examples/${model_type}" - python3 convert_checkpoint.py \ - --model_dir "${model_path}" \ - --dtype "${model_dtype}" \ - --tp_size "${model_tp_size}" \ - --output_dir "${trt_model_path}" - trtllm-build \ - --checkpoint_dir "${trt_model_path}" \ - --use_fused_mlp \ - --reduce_fusion disable \ - --workers 8 \ - --gpt_attention_plugin "${model_dtype}" \ - --gemm_plugin "${model_dtype}" \ - --tp_size "${model_tp_size}" \ - --max_batch_size "${max_batch_size}" \ - --max_input_len "${max_input_len}" \ - --max_seq_len "${max_seq_len}" \ - --max_num_tokens "${max_num_tokens}" \ - --output_dir "${trt_engine_path}" - - # handle triton protobuf files and launch triton server - cd /tensorrtllm_backend - mkdir triton_model_repo - cp -r all_models/inflight_batcher_llm/* triton_model_repo/ - cd triton_model_repo - rm -rf ./tensorrt_llm/1/* - cp -r "${trt_engine_path}"/* ./tensorrt_llm/1 - python3 ../tools/fill_template.py -i tensorrt_llm/config.pbtxt triton_backend:tensorrtllm,engine_dir:/tensorrtllm_backend/triton_model_repo/tensorrt_llm/1,decoupled_mode:true,batching_strategy:inflight_fused_batching,batch_scheduler_policy:guaranteed_no_evict,exclude_input_in_output:true,triton_max_batch_size:2048,max_queue_delay_microseconds:0,max_beam_width:1,max_queue_size:2048,enable_kv_cache_reuse:false - python3 ../tools/fill_template.py -i preprocessing/config.pbtxt "triton_max_batch_size:2048,tokenizer_dir:$model_path,preprocessing_instance_count:5" - python3 ../tools/fill_template.py -i postprocessing/config.pbtxt "triton_max_batch_size:2048,tokenizer_dir:$model_path,postprocessing_instance_count:5,skip_special_tokens:false" - python3 ../tools/fill_template.py -i ensemble/config.pbtxt triton_max_batch_size:"$max_batch_size" - python3 ../tools/fill_template.py -i tensorrt_llm_bls/config.pbtxt "triton_max_batch_size:$max_batch_size,decoupled_mode:true,accumulate_tokens:False,bls_instance_count:1" - cd /tensorrtllm_backend - python3 scripts/launch_triton_server.py \ - --world_size="${model_tp_size}" \ - --model_repo=/tensorrtllm_backend/triton_model_repo & - -} - -launch_tgi_server() { - model=$(echo "$common_params" | jq -r '.model') - tp=$(echo "$common_params" | jq -r '.tp') - port=$(echo "$common_params" | jq -r '.port') - server_args=$(json2args "$server_params") - - if echo "$common_params" | jq -e 'has("fp8")' >/dev/null; then - echo "Key 'fp8' exists in common params." - server_command="/tgi-entrypoint.sh \ - --model-id $model \ - --num-shard $tp \ - --port $port \ - --quantize fp8 \ - $server_args" - else - echo "Key 'fp8' does not exist in common params." - server_command="/tgi-entrypoint.sh \ - --model-id $model \ - --num-shard $tp \ - --port $port \ - $server_args" - fi - - echo "Server command: $server_command" - eval "$server_command" & - -} - -launch_lmdeploy_server() { - model=$(echo "$common_params" | jq -r '.model') - tp=$(echo "$common_params" | jq -r '.tp') - port=$(echo "$common_params" | jq -r '.port') - server_args=$(json2args "$server_params") - - server_command="lmdeploy serve api_server $model \ - --tp $tp \ - --server-port $port \ - $server_args" - - # run the server - echo "Server command: $server_command" - bash -c "$server_command" & -} - -launch_sglang_server() { - - model=$(echo "$common_params" | jq -r '.model') - tp=$(echo "$common_params" | jq -r '.tp') - port=$(echo "$common_params" | jq -r '.port') - server_args=$(json2args "$server_params") - - if echo "$common_params" | jq -e 'has("fp8")' >/dev/null; then - echo "Key 'fp8' exists in common params. Use neuralmagic fp8 model for convenience." - model=$(echo "$common_params" | jq -r '.neuralmagic_quantized_model') - server_command="python3 \ - -m sglang.launch_server \ - --tp $tp \ - --model-path $model \ - --port $port \ - $server_args" - else - echo "Key 'fp8' does not exist in common params." - server_command="python3 \ - -m sglang.launch_server \ - --tp $tp \ - --model-path $model \ - --port $port \ - $server_args" - fi - - # run the server - echo "Server command: $server_command" - eval "$server_command" & -} - -launch_vllm_server() { - - export VLLM_HOST_IP=$(hostname -I | awk '{print $1}') - - model=$(echo "$common_params" | jq -r '.model') - tp=$(echo "$common_params" | jq -r '.tp') - port=$(echo "$common_params" | jq -r '.port') - server_args=$(json2args "$server_params") - - if echo "$common_params" | jq -e 'has("fp8")' >/dev/null; then - echo "Key 'fp8' exists in common params. Use neuralmagic fp8 model for convenience." - model=$(echo "$common_params" | jq -r '.neuralmagic_quantized_model') - server_command="vllm serve $model \ - -tp $tp \ - --port $port \ - $server_args" - else - echo "Key 'fp8' does not exist in common params." - server_command="vllm serve $model \ - -tp $tp \ - --port $port \ - $server_args" - fi - - # run the server - echo "Server command: $server_command" - eval "$server_command" & -} - -main() { - - if [[ "$CURRENT_LLM_SERVING_ENGINE" == "trt" ]]; then - launch_trt_server - fi - - if [[ "$CURRENT_LLM_SERVING_ENGINE" == "tgi" ]]; then - launch_tgi_server - fi - - if [[ "$CURRENT_LLM_SERVING_ENGINE" == "lmdeploy" ]]; then - launch_lmdeploy_server - fi - - if [[ "$CURRENT_LLM_SERVING_ENGINE" == "sglang" ]]; then - launch_sglang_server - fi - - if [[ "$CURRENT_LLM_SERVING_ENGINE" == *"vllm"* ]]; then - launch_vllm_server - fi -} - -main diff --git a/.buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh b/.buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh deleted file mode 100644 index 91032978ec..0000000000 --- a/.buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh +++ /dev/null @@ -1,896 +0,0 @@ -#!/bin/bash -# This script assumes that we are already inside the vllm/ directory -# Benchmarking results will be available inside vllm/benchmarks/results/ - -# Do not set -e, as the mixtral 8x22B model tends to crash occasionally -# and we still want to see other benchmarking results even when mixtral crashes. -set -x -set -o pipefail - -# Environment-driven debug controls (like ON_CPU=1) -DRY_RUN="${DRY_RUN:-0}" -MODEL_FILTER="${MODEL_FILTER:-}" -DTYPE_FILTER="${DTYPE_FILTER:-}" - -# Adaptive search controls -ENABLE_ADAPTIVE_CONCURRENCY="${ENABLE_ADAPTIVE_CONCURRENCY:-0}" -SLA_TTFT_MS="${SLA_TTFT_MS:-3000}" -SLA_TPOT_MS="${SLA_TPOT_MS:-100}" -ADAPTIVE_MAX_PROBES="${ADAPTIVE_MAX_PROBES:-8}" -ADAPTIVE_MAX_CONCURRENCY="${ADAPTIVE_MAX_CONCURRENCY:-1024}" - -check_gpus() { - if command -v nvidia-smi; then - # check the number of GPUs and GPU type. - declare -g gpu_count=$(nvidia-smi --list-gpus | grep -c . || true) - elif command -v amd-smi; then - declare -g gpu_count=$(amd-smi list | grep -c 'GPU' || true) - elif command -v hl-smi; then - declare -g gpu_count=$(hl-smi --list | grep -ci "Module ID" || true) - fi - - if [[ $gpu_count -gt 0 ]]; then - echo "GPU found." - else - echo "Need at least 1 GPU to run benchmarking." - exit 1 - fi - - declare -g arch_suffix='' - - if command -v nvidia-smi; then - declare -g gpu_type=$(nvidia-smi --query-gpu=name --format=csv,noheader | awk '{print $2}') - elif command -v amd-smi; then - declare -g gpu_type=$(amd-smi static -g 0 -a | grep 'MARKET_NAME' | awk '{print $2}') - elif command -v hl-smi; then - declare -g gpu_type=$(hl-smi -q | grep "Product Name" | head -n 1 | awk -F ':' '{print $2}' | sed 's/^ *//') - arch_suffix='-hpu' - fi - echo "GPU type is $gpu_type" -} - -check_cpus() { - # check the number of CPUs and NUMA Node and GPU type. - declare -g numa_count=$(lscpu | grep "NUMA node(s):" | awk '{print $3}') - if [[ $numa_count -gt 0 ]]; then - echo "NUMA found." - echo "$numa_count" - else - echo "Need at least 1 NUMA to run benchmarking." - exit 1 - fi - if [[ "$(uname -m)" == "aarch64" ]] || [[ "$(uname -m)" == "arm64" ]]; then - declare -g gpu_type="arm64-cpu" - else - declare -g gpu_type="cpu" - fi - echo "GPU type is $gpu_type" -} - -check_hf_token() { - # check if HF_TOKEN is available and valid - if [[ -z "$HF_TOKEN" ]]; then - echo "Error: HF_TOKEN is not set." - exit 1 - elif [[ ! "$HF_TOKEN" =~ ^hf_ ]]; then - echo "Error: HF_TOKEN does not start with 'hf_'." - exit 1 - else - echo "HF_TOKEN is set and valid." - fi -} - -ensure_sharegpt_downloaded() { - local FILE=ShareGPT_V3_unfiltered_cleaned_split.json - if [ ! -f "$FILE" ]; then - wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/$FILE - else - echo "$FILE already exists." - fi -} - -json2args() { - # transforms the JSON string to command line args, and '_' is replaced to '-' - # example: - # input: { "model": "meta-llama/Llama-2-7b-chat-hf", "tensor_parallel_size": 1 } - # output: --model meta-llama/Llama-2-7b-chat-hf --tensor-parallel-size 1 - local json_string=$1 - local args=$( - echo "$json_string" | jq -r ' - to_entries | - map("--" + (.key | gsub("_"; "-")) + " " + (.value | tostring)) | - join(" ") - ' - ) - echo "$args" -} - -json2envs() { - # transforms the JSON string to environment variables. - # example: - # input: { "VLLM_CPU_KVCACHE_SPACE": 5 } - # output: VLLM_CPU_KVCACHE_SPACE=5 - local json_string=$1 - local args=$( - echo "$json_string" | jq -r ' - to_entries | - map((.key ) + "=" + (.value | tostring)) | - join(" ") - ' - ) - echo "$args" -} - -wait_for_server() { - local timeout_val="1200" - timeout "$timeout_val" bash -c ' - until curl -sf http://localhost:8000/v1/models >/dev/null; do - sleep 1 - done - ' -} - -kill_processes_launched_by_current_bash() { - # Kill all python processes launched from current bash script - current_shell_pid=$$ - processes=$(ps -eo pid,ppid,command | awk -v ppid="$current_shell_pid" -v proc="$1" '$2 == ppid && $3 ~ proc {print $1}') - if [ -n "$processes" ]; then - echo "Killing the following processes matching '$1':" - echo "$processes" - echo "$processes" | xargs kill -9 - else - echo "No processes found matching '$1'." - fi -} - -kill_gpu_processes() { - - ps -aux - lsof -t -i:8000 | xargs -r kill -9 - pgrep python3 | xargs -r kill -9 - # vLLM now names the process with VLLM prefix after https://github.com/vllm-project/vllm/pull/21445 - pgrep VLLM | xargs -r kill -9 - - # wait until GPU memory usage smaller than 1GB - if command -v nvidia-smi; then - while [ "$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -n 1)" -ge 1000 ]; do - sleep 1 - done - elif command -v amd-smi; then - while [ "$(amd-smi metric -g 0 | grep 'USED_VRAM' | awk '{print $2}')" -ge 1000 ]; do - sleep 1 - done - elif command -v hl-smi; then - while [ "$(hl-smi -q | grep "Used" | head -n 1 | awk '{print $3}')" -ge 1000 ]; do - sleep 1 - done - fi - - # remove vllm config file - rm -rf ~/.config/vllm - -} - -upload_to_buildkite() { - # upload the benchmarking results to buildkite - - # if the agent binary is not found, skip uploading the results, exit 0 - # Check if buildkite-agent is available in the PATH or at /workspace/buildkite-agent - if command -v buildkite-agent >/dev/null 2>&1; then - BUILDKITE_AGENT_COMMAND="buildkite-agent" - elif [ -f /workspace/buildkite-agent ]; then - BUILDKITE_AGENT_COMMAND="/workspace/buildkite-agent" - else - echo "buildkite-agent binary not found. Skip uploading the results." - return 0 - fi - - # Use the determined command to annotate and upload artifacts - $BUILDKITE_AGENT_COMMAND annotate --style "info" --context "$BUILDKITE_LABEL-benchmark-results" < "$RESULTS_FOLDER/benchmark_results.md" - $BUILDKITE_AGENT_COMMAND artifact upload "$RESULTS_FOLDER/*" -} - -# ------------------------------- -# Adaptive concurrency helpers -# ------------------------------- -result_json_path_for_serving() { - local test_name=$1 - local qps=$2 - local max_concurrency=$3 - echo "$RESULTS_FOLDER/${test_name}_qps_${qps}_concurrency_${max_concurrency}.json" -} - -extract_metric_ms() { - local metric_name=$1 - local json_file=$2 - - [[ -f "$json_file" ]] || return 0 - - if [[ "$metric_name" == "ttft" ]]; then - jq -r ' - [ - .ttft_ms.p99?, - .metrics.ttft_ms.p99?, - .ttft.p99?, - .metrics.ttft.p99?, - .p99_ttft_ms?, - .ttft_ms.mean?, - .metrics.ttft_ms.mean?, - .ttft.mean?, - .metrics.ttft.mean?, - .mean_ttft_ms? - ] | map(select(. != null)) | .[0] // empty - ' "$json_file" - else - jq -r ' - [ - .tpot_ms.p99?, - .metrics.tpot_ms.p99?, - .tpot.p99?, - .metrics.tpot.p99?, - .p99_tpot_ms?, - .itl_ms.p99?, - .metrics.itl_ms.p99?, - .inter_token_latency_ms.p99?, - .tpot_ms.mean?, - .metrics.tpot_ms.mean?, - .tpot.mean?, - .metrics.tpot.mean?, - .itl_ms.mean?, - .metrics.itl_ms.mean?, - .mean_tpot_ms?, - .mean_itl_ms? - ] | map(select(. != null)) | .[0] // empty - ' "$json_file" - fi -} - -evaluate_sla_from_json() { - local json_file=$1 - local ttft - local tpot - local pass - - [[ -f "$json_file" ]] || return 2 - - ttft=$(extract_metric_ms ttft "$json_file") - tpot=$(extract_metric_ms tpot "$json_file") - - [[ -n "$ttft" && -n "$tpot" ]] || return 2 - - pass=$(jq -n \ - --argjson ttft "$ttft" \ - --argjson tpot "$tpot" \ - --argjson sla_ttft "$SLA_TTFT_MS" \ - --argjson sla_tpot "$SLA_TPOT_MS" \ - '($ttft <= $sla_ttft) and ($tpot <= $sla_tpot)') - - [[ "$pass" == "true" ]] -} - -write_adaptive_summary_json() { - local summary_file=$1 - local test_name=$2 - local qps=$3 - local static_last_pass=$4 - local static_first_fail=$5 - local final_last_pass=$6 - local final_first_fail=$7 - - jq -n \ - --arg test_name "$test_name" \ - --arg qps "$qps" \ - --argjson sla_ttft "$SLA_TTFT_MS" \ - --argjson sla_tpot "$SLA_TPOT_MS" \ - --arg static_last_pass "${static_last_pass:-}" \ - --arg static_first_fail "${static_first_fail:-}" \ - --arg final_last_pass "${final_last_pass:-}" \ - --arg final_first_fail "${final_first_fail:-}" \ - '{ - test_name: $test_name, - qps: $qps, - sla_ttft_ms: $sla_ttft, - sla_tpot_ms: $sla_tpot, - static_last_pass: (if $static_last_pass == "" then null else ($static_last_pass | tonumber) end), - static_first_fail: (if $static_first_fail == "" then null else ($static_first_fail | tonumber) end), - final_last_pass: (if $final_last_pass == "" then null else ($final_last_pass | tonumber) end), - final_first_fail: (if $final_first_fail == "" then null else ($final_first_fail | tonumber) end) - }' > "$summary_file" -} - -run_single_serving_probe() { - local test_name=$1 - local qps=$2 - local max_concurrency=$3 - local tp=$4 - local compilation_config_mode=$5 - local optimization_level=$6 - local client_args_effective=$7 - local client_remote_args=$8 - local server_command=$9 - - local new_test_name="${test_name}_qps_${qps}_concurrency_${max_concurrency}" - local result_json - local num_prompts_arg="" - local client_command - - result_json=$(result_json_path_for_serving "$test_name" "$qps" "$max_concurrency") - - if [[ -f "$result_json" ]]; then - evaluate_sla_from_json "$result_json" - return $? - fi - - if [[ -n "${PROMPTS_PER_CONCURRENCY}" ]]; then - num_prompts=$(( max_concurrency * PROMPTS_PER_CONCURRENCY )) - if (( num_prompts < MIN_NUM_PROMPTS )); then num_prompts=$MIN_NUM_PROMPTS; fi - if (( num_prompts > MAX_NUM_PROMPTS )); then num_prompts=$MAX_NUM_PROMPTS; fi - num_prompts_arg="--num-prompts $num_prompts" - fi - - client_command="vllm bench serve \ - --save-result \ - --result-dir $RESULTS_FOLDER \ - --result-filename ${new_test_name}.json \ - --request-rate $qps \ - --max-concurrency $max_concurrency \ - $num_prompts_arg \ - --metadata tensor_parallel_size=$tp compilation_config.mode=$compilation_config_mode optimization_level=$optimization_level adaptive_search=1 \ - $client_args_effective $client_remote_args " - - echo "Adaptive probe: $client_command" - - if [[ "${DRY_RUN:-0}" != "1" ]]; then - bash -c "$client_command" - fi - - jq_output=$(jq -n \ - --arg server "$server_command" \ - --arg client "$client_command" \ - --arg gpu "$gpu_type" \ - '{ - server_command: $server, - client_command: $client, - gpu_type: $gpu, - adaptive_search: true - }') - echo "$jq_output" > "$RESULTS_FOLDER/${new_test_name}.commands" - - evaluate_sla_from_json "$result_json" -} - -adaptive_refine_from_static_results() { - local test_name=$1 - local qps=$2 - local max_concurrency_list_raw=$3 - local tp=$4 - local compilation_config_mode=$5 - local optimization_level=$6 - local client_args_effective=$7 - local client_remote_args=$8 - local server_command=$9 - - local sorted_points - local point - local rc - local static_last_pass="" - local static_first_fail="" - local largest_static="" - local step_hint=1 - local previous_point="" - local low - local high - local mid - local probes=0 - local summary_file="$RESULTS_FOLDER/${test_name}_qps_${qps}_sla_summary.json" - - [[ "${ENABLE_ADAPTIVE_CONCURRENCY}" == "1" ]] || return 0 - [[ "${DRY_RUN:-0}" != "1" ]] || return 0 - - sorted_points=$(for point in $max_concurrency_list_raw; do printf '%s\n' "$point"; done | tr -d "'" | awk '/^[0-9]+$/' | sort -n | uniq) - [[ -n "$sorted_points" ]] || return 0 - - while read -r point; do - [[ -z "$point" ]] && continue - largest_static="$point" - evaluate_sla_from_json "$(result_json_path_for_serving "$test_name" "$qps" "$point")" - rc=$? - if (( rc == 0 )); then - static_last_pass="$point" - elif (( rc == 1 )); then - if [[ -n "$static_last_pass" ]]; then - static_first_fail="$point" - break - fi - fi - - if [[ -n "$previous_point" ]]; then - step_hint=$(( point - previous_point )) - if (( step_hint < 1 )); then step_hint=1; fi - fi - previous_point="$point" - done <<< "$sorted_points" - - if [[ -z "$static_last_pass" ]]; then - write_adaptive_summary_json "$summary_file" "$test_name" "$qps" "" "$static_first_fail" "" "$static_first_fail" - return 0 - fi - - if [[ -n "$static_first_fail" ]]; then - low=$static_last_pass - high=$static_first_fail - while (( low + 1 < high )) && (( probes < ADAPTIVE_MAX_PROBES )); do - mid=$(( (low + high) / 2 )) - probes=$(( probes + 1 )) - run_single_serving_probe \ - "$test_name" "$qps" "$mid" "$tp" \ - "$compilation_config_mode" "$optimization_level" \ - "$client_args_effective" "$client_remote_args" "$server_command" - rc=$? - if (( rc == 0 )); then - low=$mid - elif (( rc == 1 )); then - high=$mid - else - break - fi - done - write_adaptive_summary_json "$summary_file" "$test_name" "$qps" "$static_last_pass" "$static_first_fail" "$low" "$high" - return 0 - fi - - low=$largest_static - high="" - while (( probes < ADAPTIVE_MAX_PROBES )); do - point=$(( low + step_hint )) - if (( point > ADAPTIVE_MAX_CONCURRENCY )); then - point=$ADAPTIVE_MAX_CONCURRENCY - fi - (( point > low )) || break - probes=$(( probes + 1 )) - run_single_serving_probe \ - "$test_name" "$qps" "$point" "$tp" \ - "$compilation_config_mode" "$optimization_level" \ - "$client_args_effective" "$client_remote_args" "$server_command" - rc=$? - if (( rc == 0 )); then - low=$point - (( point == ADAPTIVE_MAX_CONCURRENCY )) && break - step_hint=$(( step_hint * 2 )) - if (( step_hint < 1 )); then step_hint=1; fi - elif (( rc == 1 )); then - high=$point - break - else - break - fi - done - - if [[ -n "$high" ]]; then - while (( low + 1 < high )) && (( probes < ADAPTIVE_MAX_PROBES )); do - mid=$(( (low + high) / 2 )) - probes=$(( probes + 1 )) - run_single_serving_probe \ - "$test_name" "$qps" "$mid" "$tp" \ - "$compilation_config_mode" "$optimization_level" \ - "$client_args_effective" "$client_remote_args" "$server_command" - rc=$? - if (( rc == 0 )); then - low=$mid - elif (( rc == 1 )); then - high=$mid - else - break - fi - done - fi - - write_adaptive_summary_json "$summary_file" "$test_name" "$qps" "$static_last_pass" "" "$low" "$high" -} - -run_benchmark_tests() { - # run benchmark tests using `vllm bench ` command - # $1: test type (latency or throughput) - # $2: a json file specifying test cases - - local test_type=$1 - local test_file=$2 - - # Iterate over tests - jq -c '.[]' "$test_file" | while read -r params; do - # get the test name, and append the GPU type back to it. - test_name=$(echo "$params" | jq -r '.test_name') - if [[ ! "$test_name" =~ ^${test_type}_ ]]; then - echo "In ${test_type}-test.json, test_name must start with \"${test_type}_\"." - exit 1 - fi - - # if TEST_SELECTOR is set, only run the test cases that match the selector - if [[ -n "$TEST_SELECTOR" ]] && [[ ! "$test_name" =~ $TEST_SELECTOR ]]; then - echo "Skip test case $test_name." - continue - fi - - # get arguments - bench_params=$(echo "$params" | jq -r '.parameters') - bench_args=$(json2args "$bench_params") - bench_environment_variables=$(echo "$params" | jq -r '.environment_variables') - bench_envs=$(json2envs "$bench_environment_variables") - - # check if there is enough GPU to run the test - tp=$(echo "$bench_params" | jq -r '.tensor_parallel_size') - if [[ "$ON_CPU" == "1" ]]; then - pp=$(echo "$bench_params" | jq -r '.pipeline_parallel_size // 1') - world_size=$(($tp*$pp)) - if [[ $numa_count -lt $world_size && -z "${REMOTE_HOST}" ]]; then - echo "Required world-size $world_size but only $numa_count NUMA nodes found. Skip testcase $test_name." - continue - fi - else - if [[ $gpu_count -lt $tp ]]; then - echo "Required tensor-parallel-size $tp but only $gpu_count GPU found. Skip testcase $test_name." - continue - fi - fi - - bench_command=" $bench_envs vllm bench $test_type \ - --output-json $RESULTS_FOLDER/${test_name}.json \ - $bench_args" - - echo "Running test case $test_name" - echo "${test_type^} command: $bench_command" - - # recording benchmarking command and GPU command - jq_output=$(jq -n \ - --arg command "$bench_command" \ - --arg gpu "$gpu_type" \ - --arg test_type "$test_type" \ - '{ - ($test_type + "_command"): $command, - gpu_type: $gpu - }') - echo "$jq_output" >"$RESULTS_FOLDER/$test_name.commands" - - # run the benchmark - eval "$bench_command" - - kill_gpu_processes - - done -} - -run_latency_tests() { run_benchmark_tests "latency" "$1"; } -run_startup_tests() { run_benchmark_tests "startup" "$1"; } -run_throughput_tests() { run_benchmark_tests "throughput" "$1"; } - -merge_serving_tests_stream() { - # Emit merged serving test objects, optionally filtered by MODEL_FILTER/DTYPE_FILTER in DRY_RUN mode. - # This helper does NOT modify JSON; it only filters the stream in dry-run mode. - local serving_test_file="$1" - # shellcheck disable=SC2016 - local merged=' - if type == "array" then - # Plain format: test cases array - .[] - elif (type == "object" and has("tests")) then - # merge the default parameters into each test cases - . as $root - | ($root.defaults // {}) as $d - | ($root.tests // [])[] - # default qps / max_concurrency from defaults if missing - | .qps_list = (.qps_list // $d.qps_list) - | .max_concurrency_list = (.max_concurrency_list // $d.max_concurrency_list) - # merge envs / params: test overrides defaults - | .server_environment_variables = - (($d.server_environment_variables // {}) + (.server_environment_variables // {})) - | .server_parameters = - (($d.server_parameters // {}) + (.server_parameters // {})) - | .client_parameters = - (($d.client_parameters // {}) + (.client_parameters // {})) - else - error("Unsupported serving test file format: must be array or object with .tests") - end - ' - - jq -c "$merged" "$serving_test_file" | \ - if [[ "${DRY_RUN:-0}" == "1" && ( "${MODEL_FILTER}${DTYPE_FILTER}" != "" ) ]]; then - jq -c --arg model "$MODEL_FILTER" --arg dtype "$DTYPE_FILTER" ' - select((($model|length)==0) - or ((.server_parameters.model // "") == $model) - or ((.client_parameters.model // "") == $model)) - | select((($dtype|length)==0) or ((.server_parameters.dtype // "") == $dtype)) - ' - else - cat - fi -} - -run_serving_tests() { - # run serving tests using `vllm bench serve` command - # $1: a json file specifying serving test cases - # - # Supported JSON formats: - # 1) Plain format: top-level array - # [ { "test_name": "...", "server_parameters": {...}, ... }, ... ] - # - # 2) Default parameters field + plain format tests - # { - # "defaults": { ... }, - # "tests": [ { "test_name": "...", "server_parameters": {...}, ... }, ... ] - # } - - local serving_test_file - serving_test_file=$1 - - # In dry-run mode, if filters are provided but no tests match, fail fast. - if [[ "${DRY_RUN:-0}" == "1" && ( "${MODEL_FILTER}${DTYPE_FILTER}" != "" ) ]]; then - local count - count=$(merge_serving_tests_stream "$serving_test_file" | wc -l | tr -d ' ') - if [[ "$count" -eq 0 ]]; then - echo "No matching serving tests found in $serving_test_file for model='$MODEL_FILTER' dtype='$DTYPE_FILTER'." >&2 - return 0 - fi - fi - - # Iterate over serving tests (merged + optional filtered stream) - merge_serving_tests_stream "$serving_test_file" | while read -r params; do - # get the test name, and append the GPU type back to it. - test_name=$(echo "$params" | jq -r '.test_name') - if [[ ! "$test_name" =~ ^serving_ ]]; then - echo "In serving-test.json, test_name must start with \"serving_\"." - exit 1 - fi - - # if TEST_SELECTOR is set, only run the test cases that match the selector - if [[ -n "$TEST_SELECTOR" ]] && [[ ! "$test_name" =~ $TEST_SELECTOR ]]; then - echo "Skip test case $test_name." - continue - fi - - # get client and server arguments (after merged the default parameters) - server_params=$(echo "$params" | jq -r '.server_parameters') - server_envs=$(echo "$params" | jq -r '.server_environment_variables') - client_params=$(echo "$params" | jq -r '.client_parameters') - - # vLLM serve CLI: model must be positional (no --model). Convert server_parameters accordingly. - server_model=$(echo "$server_params" | jq -r '.model // empty') - if [[ -z "$server_model" || "$server_model" == "null" ]]; then - echo "Error: serving test '$test_name' is missing server_parameters.model" >&2 - exit 1 - fi - server_params_no_model=$(echo "$server_params" | jq -c 'del(.model)') - server_args=$(json2args "$server_params_no_model") - - server_envs=$(json2envs "$server_envs") - client_args=$(json2args "$client_params") - - # ------------------------------------------------------------ - # Option 1: Dynamic num-prompts scaling based on max_concurrency - # - # If PROMPTS_PER_CONCURRENCY is set, override JSON num_prompts with: - # num_prompts = max_concurrency * PROMPTS_PER_CONCURRENCY - # - # If PROMPTS_PER_CONCURRENCY is NOT set, keep JSON num_prompts behavior - # unchanged (i.e., whatever is in serving-tests-*.json). - # ------------------------------------------------------------ - PROMPTS_PER_CONCURRENCY="${PROMPTS_PER_CONCURRENCY-}" # no default on purpose - MIN_NUM_PROMPTS="${MIN_NUM_PROMPTS:-1}" - MAX_NUM_PROMPTS="${MAX_NUM_PROMPTS:-1000000}" - - if [[ -n "${PROMPTS_PER_CONCURRENCY}" ]]; then - # Remove any fixed --num-prompts from JSON-derived args (avoid duplicates) - # Remove any fixed --num-prompts from JSON-derived args (avoid duplicates) - # Handles: --num-prompts 123 and --num-prompts=123 - client_args_no_np="$( - printf ' %s ' "$client_args" \ - | sed -E \ - -e 's/[[:space:]]--num-prompts=([^[:space:]]+)([[:space:]]|$)/ /g' \ - -e 's/[[:space:]]--num-prompts[[:space:]]+([^[:space:]]+)([[:space:]]|$)/ /g' - )" - # normalize whitespace - client_args_no_np="$(echo "$client_args_no_np" | tr -s ' ' | sed -E 's/^ //; s/ $//')" - client_args_no_np="$(echo "$client_args_no_np" | xargs)" - client_args_effective="$client_args_no_np" - else - client_args_effective="$client_args" - fi - # qps_list - qps_list=$(echo "$params" | jq -r '.qps_list') - qps_list=$(echo "$qps_list" | jq -r '.[] | @sh') - echo "Running over qps list $qps_list" - - # max_concurrency_list (fallback to num_prompts if missing) - max_concurrency_list=$(echo "$params" | jq -r '.max_concurrency_list') - if [[ -z "$max_concurrency_list" || "$max_concurrency_list" == "null" ]]; then - num_prompts=$(echo "$client_params" | jq -r '.num_prompts') - max_concurrency_list="[$num_prompts]" - fi - max_concurrency_list=$(echo "$max_concurrency_list" | jq -r '.[] | @sh') - echo "Running over max concurrency list $max_concurrency_list" - - # check if there is enough resources to run the test - tp=$(echo "$server_params" | jq -r '.tensor_parallel_size') - if [[ "$ON_CPU" == "1" ]]; then - pp=$(echo "$server_params" | jq -r '.pipeline_parallel_size // 1') - world_size=$(($tp*$pp)) - if [[ $numa_count -lt $world_size && -z "${REMOTE_HOST}" ]]; then - echo "Required world-size $world_size but only $numa_count NUMA nodes found. Skip testcase $test_name." - continue - fi - else - if [[ $gpu_count -lt $tp ]]; then - echo "Required tensor-parallel-size $tp but only $gpu_count GPU found. Skip testcase $test_name." - continue - fi - fi - - # check if server model and client model is aligned - client_model=$(echo "$client_params" | jq -r '.model') - if [[ $server_model != "$client_model" ]]; then - echo "Server model and client model must be the same. Skip testcase $test_name." - continue - fi - - server_command="$server_envs vllm serve $server_model \ - $server_args" - - # run the server - echo "Running test case $test_name" - echo "Server command: $server_command" - # support remote vllm server - client_remote_args="" - if [[ -z "${REMOTE_HOST}" && "${DRY_RUN:-0}" != "1" ]]; then - bash -c "$server_command" & - server_pid=$! - # wait until the server is alive - if wait_for_server; then - echo "" - echo "vLLM server is up and running." - else - echo "" - echo "vLLM failed to start within the timeout period." - fi - elif [[ "${DRY_RUN:-0}" == "1" ]]; then - # dry-run: don't start server - echo "Dry Run." - else - server_command="Using Remote Server $REMOTE_HOST $REMOTE_PORT" - if [[ ${REMOTE_PORT} ]]; then - client_remote_args=" --host=$REMOTE_HOST --port=$REMOTE_PORT " - else - client_remote_args=" --host=$REMOTE_HOST " - fi - fi - - # save the compilation mode and optimization level on the serving results - # whenever they are set - compilation_config_mode=$(echo "$server_params" | jq -r '."compilation_config.mode" // empty') - optimization_level=$(echo "$server_params" | jq -r '.optimization_level // empty') - - # iterate over different QPS - for qps in $qps_list; do - # remove the surrounding single quote from qps - if [[ "$qps" == *"inf"* ]]; then - qps="inf" - fi - - # iterate over different max_concurrency - for max_concurrency in $max_concurrency_list; do - new_test_name="${test_name}_qps_${qps}_concurrency_${max_concurrency}" - echo " new test name $new_test_name" - # If PROMPTS_PER_CONCURRENCY is set, compute per-concurrency --num-prompts. - num_prompts_arg="" - if [[ -n "${PROMPTS_PER_CONCURRENCY}" ]]; then - num_prompts=$(( max_concurrency * PROMPTS_PER_CONCURRENCY )) - if (( num_prompts < MIN_NUM_PROMPTS )); then num_prompts=$MIN_NUM_PROMPTS; fi - if (( num_prompts > MAX_NUM_PROMPTS )); then num_prompts=$MAX_NUM_PROMPTS; fi - num_prompts_arg="--num-prompts $num_prompts" - fi - # pass the tensor parallel size, the compilation mode, and the optimization - # level to the client so that they can be used on the benchmark dashboard - client_command="vllm bench serve \ - --save-result \ - --result-dir $RESULTS_FOLDER \ - --result-filename ${new_test_name}.json \ - --request-rate $qps \ - --max-concurrency $max_concurrency \ - $num_prompts_arg \ - --metadata tensor_parallel_size=$tp compilation_config.mode=$compilation_config_mode optimization_level=$optimization_level \ - $client_args_effective $client_remote_args " - - echo "Running test case $test_name with qps $qps" - echo "Client command: $client_command" - - if [[ "${DRY_RUN:-0}" != "1" ]]; then - bash -c "$client_command" - fi - - # record the benchmarking commands - jq_output=$(jq -n \ - --arg server "$server_command" \ - --arg client "$client_command" \ - --arg gpu "$gpu_type" \ - '{ - server_command: $server, - client_command: $client, - gpu_type: $gpu - }') - echo "$jq_output" >"$RESULTS_FOLDER/${new_test_name}.commands" - - done - - adaptive_refine_from_static_results \ - "$test_name" "$qps" "$max_concurrency_list" "$tp" \ - "$compilation_config_mode" "$optimization_level" \ - "$client_args_effective" "$client_remote_args" "$server_command" - done - - # clean up - if [[ "${DRY_RUN:-0}" != "1" ]]; then - kill -9 "$server_pid" - kill_gpu_processes - fi - done -} - -main() { - - local ARCH - ARCH='' - if [[ "$ON_CPU" == "1" ]]; then - check_cpus - ARCH="-$gpu_type" - else - check_gpus - ARCH="$arch_suffix" - fi - - # DRY_RUN does not execute vLLM; do not require HF_TOKEN. - if [[ "${DRY_RUN:-0}" != "1" ]]; then - check_hf_token - else - echo "DRY_RUN=1 -> skip HF_TOKEN validation" - fi - - # dependencies - (which wget && which curl) || (apt-get update && apt-get install -y wget curl) - (which jq) || (apt-get update && apt-get -y install jq) - (which lsof) || (apt-get update && apt-get install -y lsof) - - # get the current IP address, required by `vllm bench serve` command - export VLLM_HOST_IP=$(hostname -I | awk '{print $1}') - # turn of the reporting of the status of each request, to clean up the terminal output - export VLLM_LOGGING_LEVEL="WARNING" - - # prepare for benchmarking - cd benchmarks || exit 1 - ensure_sharegpt_downloaded - declare -g RESULTS_FOLDER=results/ - mkdir -p $RESULTS_FOLDER - QUICK_BENCHMARK_ROOT=../.buildkite/performance-benchmarks/ - - # dump vllm info via vllm collect-env - env_output=$(vllm collect-env) - echo "$env_output" >"$RESULTS_FOLDER/vllm_env.txt" - - # benchmarking - run_serving_tests $QUICK_BENCHMARK_ROOT/tests/"${SERVING_JSON:-serving-tests$ARCH.json}" || exit $? - - if [[ "${DRY_RUN:-0}" == "1" ]]; then - echo "DRY_RUN=1 -> skip latency/startup/throughput suites" - exit 0 - fi - - run_latency_tests $QUICK_BENCHMARK_ROOT/tests/"${LATENCY_JSON:-latency-tests$ARCH.json}" - run_startup_tests $QUICK_BENCHMARK_ROOT/tests/"${STARTUP_JSON:-startup-tests$ARCH.json}" - run_throughput_tests $QUICK_BENCHMARK_ROOT/tests/"${THROUGHPUT_JSON:-throughput-tests$ARCH.json}" - - # postprocess benchmarking results - pip install tabulate pandas - python3 $QUICK_BENCHMARK_ROOT/scripts/convert-results-json-to-markdown.py - python3 $QUICK_BENCHMARK_ROOT/scripts/compare-json-results.py -f $RESULTS_FOLDER/benchmark_results.json - - upload_to_buildkite -} - -main "$@" diff --git a/.buildkite/performance-benchmarks/tests/genai-perf-tests.json b/.buildkite/performance-benchmarks/tests/genai-perf-tests.json deleted file mode 100644 index afb844880f..0000000000 --- a/.buildkite/performance-benchmarks/tests/genai-perf-tests.json +++ /dev/null @@ -1,21 +0,0 @@ -[ - { - "test_name": "llama8B_tp1_genai_perf", - "qps_list": [4,8,16,32], - "common_parameters": { - "model": "meta-llama/Meta-Llama-3-8B-Instruct", - "tp": 1, - "port": 8000, - "num_prompts": 500, - "reuse_server": false - }, - "vllm_server_parameters": { - "disable_log_stats": "", - "gpu_memory_utilization": 0.9, - "max_num_seqs": 512, - "dtype": "bfloat16" - }, - "genai_perf_input_parameters": { - } - } -] \ No newline at end of file diff --git a/.buildkite/performance-benchmarks/tests/latency-tests-arm64-cpu.json b/.buildkite/performance-benchmarks/tests/latency-tests-arm64-cpu.json deleted file mode 100644 index d0693a19dc..0000000000 --- a/.buildkite/performance-benchmarks/tests/latency-tests-arm64-cpu.json +++ /dev/null @@ -1,24 +0,0 @@ -[ - { - "test_name": "latency_llama8B_tp1", - "environment_variables": { - "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1, - "VLLM_CPU_KVCACHE_SPACE": 40 - }, - "parameters": { - "model": "meta-llama/Llama-3.1-8B-Instruct", - "tensor_parallel_size": 1, - "load_format": "dummy", - "dtype": "bfloat16", - "distributed_executor_backend": "mp", - "block_size": 128, - "trust_remote_code": "", - "disable_log_stats": "", - "enforce_eager": "", - "max_num_batched_tokens": 2048, - "max_num_seqs": 256, - "num_iters_warmup": 5, - "num_iters": 15 - } - } -] diff --git a/.buildkite/performance-benchmarks/tests/latency-tests-cpu.json b/.buildkite/performance-benchmarks/tests/latency-tests-cpu.json deleted file mode 100644 index 07e58346fc..0000000000 --- a/.buildkite/performance-benchmarks/tests/latency-tests-cpu.json +++ /dev/null @@ -1,24 +0,0 @@ -[ - { - "test_name": "latency_llama8B_tp2", - "environment_variables": { - "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1, - "VLLM_CPU_SGL_KERNEL": 1, - "VLLM_CPU_KVCACHE_SPACE": 40 - }, - "parameters": { - "model": "meta-llama/Llama-3.1-8B-Instruct", - "tensor_parallel_size": 2, - "dtype": "bfloat16", - "distributed_executor_backend": "mp", - "block_size": 128, - "trust_remote_code": "", - "disable_log_stats": "", - "enforce_eager": "", - "max_num_batched_tokens": 2048, - "max_num_seqs": 256, - "num_iters_warmup": 5, - "num_iters": 15 - } - } -] diff --git a/.buildkite/performance-benchmarks/tests/latency-tests-hpu.json b/.buildkite/performance-benchmarks/tests/latency-tests-hpu.json deleted file mode 100644 index 3b3fb4bed8..0000000000 --- a/.buildkite/performance-benchmarks/tests/latency-tests-hpu.json +++ /dev/null @@ -1,106 +0,0 @@ -[ - { - "test_name": "latency_llama8B_tp1", - "environment_variables": { - "PT_HPU_LAZY_MODE": 1, - "VLLM_CONTIGUOUS_PA": 1, - "VLLM_DEFRAG": 1 - }, - "parameters": { - "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", - "tensor_parallel_size": 1, - "load_format": "dummy", - "num-iters-warmup": 5, - "num-iters": 15, - "max-model-len": 256, - "async-scheduling": "" - } - }, - { - "test_name": "latency_llama70B_tp4", - "environment_variables": { - "PT_HPU_LAZY_MODE": 1, - "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1, - "VLLM_CONTIGUOUS_PA": 1, - "VLLM_DEFRAG": 1 - }, - "parameters": { - "model": "meta-llama/Meta-Llama-3.1-70B-Instruct", - "tensor_parallel_size": 4, - "load_format": "dummy", - "num-iters-warmup": 5, - "num-iters": 15, - "max-model-len": 256, - "async-scheduling": "" - } - }, - { - "test_name": "latency_mixtral8x7B_tp2", - "environment_variables": { - "PT_HPU_LAZY_MODE": 1, - "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1, - "VLLM_CONTIGUOUS_PA": 1, - "VLLM_DEFRAG": 1 - }, - "parameters": { - "model": "mistralai/Mixtral-8x7B-Instruct-v0.1", - "tensor_parallel_size": 2, - "load_format": "dummy", - "num-iters-warmup": 5, - "num-iters": 15, - "max-model-len": 256, - "async-scheduling": "" - } - }, - { - "test_name": "latency_deepseek_r1", - "environment_variables": { - "PT_HPU_LAZY_MODE": 1, - "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1, - "VLLM_CONTIGUOUS_PA": 1, - "VLLM_DEFRAG": 1 - }, - "parameters": { - "model": "deepseek-ai/DeepSeek-R1", - "tensor_parallel_size": 8, - "load_format": "dummy", - "max-model-len": 2048, - "dtype": "bfloat16" - } - }, - { - "test_name": "latency_llama4_maverick_17b128e_instruct_fp8", - "environment_variables": { - "PT_HPU_LAZY_MODE": 1, - "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1, - "VLLM_CONTIGUOUS_PA": 1, - "VLLM_DEFRAG": 1 - }, - "parameters": { - "model": "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", - "tensor_parallel_size": 8, - "max-model-len": 512, - "max-num-seqs": 128, - "async-scheduling": "", - "gpu-memory-utilization": 0.95, - "enable_expert_parallel": "" - } - }, - { - "test_name": "latency_qwen3_8b", - "environment_variables": { - "PT_HPU_LAZY_MODE": 1, - "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1, - "VLLM_CONTIGUOUS_PA": 1, - "VLLM_DEFRAG": 1 - }, - "parameters": { - "model": "Qwen/Qwen3-8B", - "tensor_parallel_size": 1, - "max-model-len": 2048, - "max-num-seqs": 128, - "dtype": "bfloat16", - "async-scheduling": "" - } - } -] diff --git a/.buildkite/performance-benchmarks/tests/latency-tests.json b/.buildkite/performance-benchmarks/tests/latency-tests.json deleted file mode 100644 index 7762a239f9..0000000000 --- a/.buildkite/performance-benchmarks/tests/latency-tests.json +++ /dev/null @@ -1,32 +0,0 @@ -[ - { - "test_name": "latency_llama8B_tp1", - "parameters": { - "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", - "tensor_parallel_size": 1, - "load_format": "dummy", - "num_iters_warmup": 5, - "num_iters": 15 - } - }, - { - "test_name": "latency_llama70B_tp4", - "parameters": { - "model": "meta-llama/Meta-Llama-3.1-70B-Instruct", - "tensor_parallel_size": 4, - "load_format": "dummy", - "num-iters-warmup": 5, - "num-iters": 15 - } - }, - { - "test_name": "latency_mixtral8x7B_tp2", - "parameters": { - "model": "mistralai/Mixtral-8x7B-Instruct-v0.1", - "tensor_parallel_size": 2, - "load_format": "dummy", - "num-iters-warmup": 5, - "num-iters": 15 - } - } -] diff --git a/.buildkite/performance-benchmarks/tests/nightly-tests.json b/.buildkite/performance-benchmarks/tests/nightly-tests.json deleted file mode 100644 index 423a3bfe12..0000000000 --- a/.buildkite/performance-benchmarks/tests/nightly-tests.json +++ /dev/null @@ -1,311 +0,0 @@ -[ - { - "test_name": "llama8B_tp1_sharegpt", - "qps_list": [4,8,16,32,"inf"], - "common_parameters": { - "model": "meta-llama/Meta-Llama-3-8B-Instruct", - "tp": 1, - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "num_prompts": 500, - "port": 8000, - "reuse_server": false - }, - "lmdeploy_server_parameters": { - "dtype": "bfloat16" - }, - "lmdeploy_client_parameters": { - }, - "tgi_server_parameters": { - }, - "tgi_client_parameters": { - "endpoint": "/generate_stream" - }, - "trt_server_parameters": { - "model_type": "llama", - "model_dtype": "bfloat16", - "max_batch_size": 2048, - "max_input_len": 4096, - "max_seq_len": 6144, - "max_num_tokens": 16384, - "trt_llm_version": "v0.11.0" - }, - "trt_client_parameters": { - "endpoint": "/v2/models/ensemble/generate_stream" - }, - "vllm_server_parameters": { - "disable_log_stats": "", - "gpu_memory_utilization": 0.9, - "max_num_seqs": 512, - "dtype": "bfloat16" - }, - "vllm_client_parameters": { - }, - "sglang_server_parameters": { - "disable_radix_cache": "", - "enable_torch_compile": "", - "dtype": "bfloat16" - }, - "sglang_client_parameters": { - } - }, - { - "test_name": "llama8B_tp1_sonnet_512_16", - "qps_list": [4,8,16,32,"inf"], - "common_parameters": { - "model": "meta-llama/Meta-Llama-3-8B-Instruct", - "tp": 1, - "dataset_name": "sonnet", - "dataset_path": "./sonnet_4x.txt", - "num_prompts": 500, - "port": 8000, - "sonnet_input_len": 512, - "sonnet_output_len": 16, - "sonnet_prefix_len": 50, - "reuse_server": true - }, - "lmdeploy_server_parameters": { - "dtype": "bfloat16" - }, - "lmdeploy_client_parameters": { - }, - "tgi_server_parameters": { - }, - "tgi_client_parameters": { - "endpoint": "/generate_stream" - }, - "trt_server_parameters": { - "model_type": "llama", - "model_dtype": "bfloat16", - "max_batch_size": 2048, - "max_input_len": 4096, - "max_seq_len": 6144, - "max_num_tokens": 16384, - "trt_llm_version": "v0.11.0" - }, - "trt_client_parameters": { - "endpoint": "/v2/models/ensemble/generate_stream" - }, - "vllm_server_parameters": { - "disable_log_stats": "", - "gpu_memory_utilization": 0.9, - "max_num_seqs": 512, - "dtype": "bfloat16" - }, - "vllm_client_parameters": { - }, - "sglang_server_parameters": { - "disable_radix_cache": "", - "enable_torch_compile": "", - "dtype": "bfloat16" - }, - "sglang_client_parameters": { - } - }, - { - "test_name": "llama8B_tp1_sonnet_512_256", - "qps_list": [4,8,16,32,"inf"], - "common_parameters": { - "model": "meta-llama/Meta-Llama-3-8B-Instruct", - "tp": 1, - "dataset_name": "sonnet", - "dataset_path": "./sonnet_4x.txt", - "num_prompts": 500, - "port": 8000, - "sonnet_input_len": 512, - "sonnet_output_len": 256, - "sonnet_prefix_len": 50, - "reuse_server": true - }, - "lmdeploy_server_parameters": { - "dtype": "bfloat16" - }, - "lmdeploy_client_parameters": { - }, - "tgi_server_parameters": { - }, - "tgi_client_parameters": { - "endpoint": "/generate_stream" - }, - "trt_server_parameters": { - "model_type": "llama", - "model_dtype": "bfloat16", - "max_batch_size": 2048, - "max_input_len": 4096, - "max_seq_len": 6144, - "max_num_tokens": 16384, - "trt_llm_version": "v0.11.0" - }, - "trt_client_parameters": { - "endpoint": "/v2/models/ensemble/generate_stream" - }, - "vllm_server_parameters": { - "disable_log_stats": "", - "gpu_memory_utilization": 0.9, - "max_num_seqs": 512, - "dtype": "bfloat16" - }, - "vllm_client_parameters": { - }, - "sglang_server_parameters": { - "disable_radix_cache": "", - "enable_torch_compile": "", - "dtype": "bfloat16" - }, - "sglang_client_parameters": { - } - }, - { - "test_name": "llama70B_tp4_sharegpt", - "qps_list": [4,8,16,32,"inf"], - "common_parameters": { - "model": "meta-llama/Meta-Llama-3-70B-Instruct", - "tp": 4, - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "num_prompts": 500, - "port": 8000, - "reuse_server": false - }, - "lmdeploy_server_parameters": { - "dtype": "bfloat16" - }, - "lmdeploy_client_parameters": { - }, - "tgi_server_parameters": { - }, - "tgi_client_parameters": { - "endpoint": "/generate_stream" - }, - "trt_server_parameters": { - "model_type": "llama", - "model_dtype": "bfloat16", - "max_batch_size": 2048, - "max_input_len": 4096, - "max_seq_len": 6144, - "max_num_tokens": 16384, - "trt_llm_version": "v0.11.0" - }, - "trt_client_parameters": { - "endpoint": "/v2/models/ensemble/generate_stream" - }, - "vllm_server_parameters": { - "disable_log_stats": "", - "gpu_memory_utilization": 0.9, - "max_num_seqs": 512, - "dtype": "bfloat16" - }, - "vllm_client_parameters": { - }, - "sglang_server_parameters": { - "disable_radix_cache": "", - "dtype": "bfloat16" - }, - "sglang_client_parameters": { - } - }, - { - "test_name": "llama70B_tp4_sonnet_512_16", - "qps_list": [4,8,16,32,"inf"], - "common_parameters": { - "model": "meta-llama/Meta-Llama-3-70B-Instruct", - "tp": 4, - "dataset_name": "sonnet", - "dataset_path": "./sonnet_4x.txt", - "num_prompts": 500, - "port": 8000, - "sonnet_input_len": 512, - "sonnet_output_len": 16, - "sonnet_prefix_len": 50, - "reuse_server": true - }, - "lmdeploy_server_parameters": { - "dtype": "bfloat16" - }, - "lmdeploy_client_parameters": { - }, - "tgi_server_parameters": { - }, - "tgi_client_parameters": { - "endpoint": "/generate_stream" - }, - "trt_server_parameters": { - "model_type": "llama", - "model_dtype": "bfloat16", - "max_batch_size": 2048, - "max_input_len": 4096, - "max_seq_len": 6144, - "max_num_tokens": 16384, - "trt_llm_version": "v0.11.0" - }, - "trt_client_parameters": { - "endpoint": "/v2/models/ensemble/generate_stream" - }, - "vllm_server_parameters": { - "disable_log_stats": "", - "gpu_memory_utilization": 0.9, - "max_num_seqs": 512, - "dtype": "bfloat16" - }, - "vllm_client_parameters": { - }, - "sglang_server_parameters": { - "disable_radix_cache": "", - "dtype": "bfloat16" - }, - "sglang_client_parameters": { - } - }, - { - "test_name": "llama70B_tp4_sonnet_512_256", - "qps_list": [4,8,16,32,"inf"], - "common_parameters": { - "model": "meta-llama/Meta-Llama-3-70B-Instruct", - "tp": 4, - "dataset_name": "sonnet", - "dataset_path": "./sonnet_4x.txt", - "num_prompts": 500, - "port": 8000, - "sonnet_input_len": 512, - "sonnet_output_len": 256, - "sonnet_prefix_len": 50, - "reuse_server": true - }, - "lmdeploy_server_parameters": { - "dtype": "bfloat16" - }, - "lmdeploy_client_parameters": { - }, - "tgi_server_parameters": { - }, - "tgi_client_parameters": { - "endpoint": "/generate_stream" - }, - "trt_server_parameters": { - "model_type": "llama", - "model_dtype": "bfloat16", - "max_batch_size": 2048, - "max_input_len": 4096, - "max_seq_len": 6144, - "max_num_tokens": 16384, - "trt_llm_version": "v0.11.0" - }, - "trt_client_parameters": { - "endpoint": "/v2/models/ensemble/generate_stream" - }, - "vllm_server_parameters": { - "disable_log_stats": "", - "gpu_memory_utilization": 0.9, - "max_num_seqs": 512, - "dtype": "bfloat16" - }, - "vllm_client_parameters": { - }, - "sglang_server_parameters": { - "disable_radix_cache": "", - "dtype": "bfloat16" - }, - "sglang_client_parameters": { - } - } -] \ No newline at end of file diff --git a/.buildkite/performance-benchmarks/tests/serving-tests-arm64-cpu.json b/.buildkite/performance-benchmarks/tests/serving-tests-arm64-cpu.json deleted file mode 100644 index d68837fa96..0000000000 --- a/.buildkite/performance-benchmarks/tests/serving-tests-arm64-cpu.json +++ /dev/null @@ -1,129 +0,0 @@ -{ - "defaults": { - "qps_list": [ - "inf" - ], - "max_concurrency_list": [ - 12, - 16, - 24, - 32, - 64, - 128, - 200 - ], - "server_environment_variables": { - "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1, - "VLLM_CPU_SGL_KERNEL": 1, - "VLLM_CPU_KVCACHE_SPACE": 40 - }, - "server_parameters": { - "model": "meta-llama/Llama-3.1-8B-Instruct", - "tensor_parallel_size": 1, - "dtype": "bfloat16", - "distributed_executor_backend": "mp", - "block_size": 128, - "trust_remote_code": "", - "disable_log_stats": "", - "enforce_eager": "", - "max_num_batched_tokens": 2048, - "max_num_seqs": 256, - "load_format": "dummy" - }, - "client_parameters": { - "model": "meta-llama/Llama-3.1-8B-Instruct", - "backend": "vllm", - "ignore-eos": "", - "temperature": 0, - "num_prompts": 200 - } - }, - "tests": [ - { - "test_name": "serving_llama8B_tp1_sharegpt", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": { - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json" - } - }, - { - "test_name": "serving_llama8B_tp2_sharegpt", - "server_parameters": { - "tensor_parallel_size": 2 - }, - "client_parameters": { - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json" - } - }, - { - "test_name": "serving_llama8B_tp1_random_128_128", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_llama8B_tp2_random_128_128", - "server_parameters": { - "tensor_parallel_size": 2 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_llama8B_tp1_random_128_2048", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 2048 - } - }, - { - "test_name": "serving_llama8B_tp2_random_128_2048", - "server_parameters": { - "tensor_parallel_size": 2 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 2048 - } - }, - { - "test_name": "serving_llama8B_tp1_random_2048_128", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 2048, - "random-output-len": 128 - } - }, - { - "test_name": "serving_llama8B_tp2_random_2048_128", - "server_parameters": { - "tensor_parallel_size": 2 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 2048, - "random-output-len": 128 - } - } - ] -} diff --git a/.buildkite/performance-benchmarks/tests/serving-tests-cpu-asr.json b/.buildkite/performance-benchmarks/tests/serving-tests-cpu-asr.json deleted file mode 100644 index 756f0f77f2..0000000000 --- a/.buildkite/performance-benchmarks/tests/serving-tests-cpu-asr.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "defaults": { - "qps_list": [ - "inf" - ], - "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200], - "server_parameters": { - "dtype": "bfloat16", - "model": "openai/whisper-large-v3-turbo" - }, - "client_parameters": { - "model": "openai/whisper-large-v3-turbo", - "backend": "openai-audio", - "endpoint": "/v1/audio/transcriptions", - "dataset_name": "hf", - "dataset_path": "openslr/librispeech_asr", - "hf_subset": "clean", - "hf_split": "test", - "no_stream": "", - "no_oversample": "", - "temperature": 0, - "num_prompts": 200 - } - }, - "tests": [ - { - "test_name": "serving_whisper_large_v3_turbo_librispeech_clean_tp1", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": {} - } - ] -} diff --git a/.buildkite/performance-benchmarks/tests/serving-tests-cpu-embed.json b/.buildkite/performance-benchmarks/tests/serving-tests-cpu-embed.json deleted file mode 100644 index 1be4c653f1..0000000000 --- a/.buildkite/performance-benchmarks/tests/serving-tests-cpu-embed.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "defaults": { - "qps_list": [ - "inf" - ], - "max_concurrency_list": [ - 32, - 64, - 128 - ], - "server_environment_variables": { - "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1, - "VLLM_CPU_SGL_KERNEL": 1, - "VLLM_CPU_KVCACHE_SPACE": 40 - }, - "server_parameters": { - "dtype": "bfloat16", - "model": "jinaai/jina-embeddings-v3", - "trust_remote_code": "" - }, - "client_parameters": { - "model": "jinaai/jina-embeddings-v3", - "backend": "openai-embeddings", - "endpoint": "/v1/embeddings", - "dataset_name": "sharegpt", - "dataset_path": "ShareGPT_V3_unfiltered_cleaned_split.json", - "num_prompts": 200 - } - }, - "tests": [ - { - "test_name": "serving_jina_embed_v3_tp1_sharegpt", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": {} - } - ] -} diff --git a/.buildkite/performance-benchmarks/tests/serving-tests-cpu-text.json b/.buildkite/performance-benchmarks/tests/serving-tests-cpu-text.json deleted file mode 100644 index acc3669dff..0000000000 --- a/.buildkite/performance-benchmarks/tests/serving-tests-cpu-text.json +++ /dev/null @@ -1,287 +0,0 @@ -{ - "defaults": { - "qps_list": [ - "inf" - ], - "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200], - "server_environment_variables": { - "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1, - "VLLM_CPU_SGL_KERNEL": 1, - "VLLM_CPU_KVCACHE_SPACE": 40 - }, - "server_parameters": { - "model": "meta-llama/Llama-3.1-8B-Instruct", - "tensor_parallel_size": 1, - "dtype": "bfloat16", - "distributed_executor_backend": "mp", - "block_size": 128, - "trust_remote_code": "", - "disable_log_stats": "", - "max_num_batched_tokens": 2048, - "max_num_seqs": 256 - }, - "client_parameters": { - "model": "meta-llama/Llama-3.1-8B-Instruct", - "backend": "vllm", - "ignore-eos": "", - "temperature": 0, - "num_prompts": 200 - } - }, - "tests": [ - { - "test_name": "serving_llama8B_tp1_random_128_128", - "server_parameters": { - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_llama8B_int4_tp1_random_128_128", - "server_parameters": { - "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4" - }, - "client_parameters": { - "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4", - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_llama8B_int8_tp1_random_128_128", - "server_parameters": { - "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8" - }, - "client_parameters": { - "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8", - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_llama1B_tp1_random_128_128", - "server_parameters": { - "model": "meta-llama/Llama-3.2-1B" - }, - "client_parameters": { - "model": "meta-llama/Llama-3.2-1B", - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_llama3B_tp1_random_128_128", - "server_parameters": { - "model": "meta-llama/Llama-3.2-3B-Instruct" - }, - "client_parameters": { - "model": "meta-llama/Llama-3.2-3B-Instruct", - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_llama70B_tp1_random_128_128", - "server_parameters": { - "model": "meta-llama/Llama-3.3-70B-Instruct" - }, - "client_parameters": { - "model": "meta-llama/Llama-3.3-70B-Instruct", - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_granite2B_tp1_random_128_128", - "server_parameters": { - "model": "ibm-granite/granite-3.2-2b-instruct" - }, - "client_parameters": { - "model": "ibm-granite/granite-3.2-2b-instruct", - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_qwen1.7B_tp1_random_128_128", - "server_parameters": { - "model": "Qwen/Qwen3-1.7B" - }, - "client_parameters": { - "model": "Qwen/Qwen3-1.7B", - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_qwen4B_tp1_random_128_128", - "server_parameters": { - "model": "Qwen/Qwen3-4B" - }, - "client_parameters": { - "model": "Qwen/Qwen3-4B", - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_qwen8B_tp1_random_128_128", - "server_parameters": { - "model": "Qwen/Qwen3-8B" - }, - "client_parameters": { - "model": "Qwen/Qwen3-8B", - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_qwen14B_tp1_random_128_128", - "server_parameters": { - "model": "Qwen/Qwen3-14B" - }, - "client_parameters": { - "model": "Qwen/Qwen3-14B", - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_qwen30B_tp1_random_128_128", - "server_parameters": { - "model": "Qwen/Qwen3-30B-A3B" - }, - "client_parameters": { - "model": "Qwen/Qwen3-30B-A3B", - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_glm9B_tp1_random_128_128", - "server_parameters": { - "model": "zai-org/glm-4-9b-hf" - }, - "client_parameters": { - "model": "zai-org/glm-4-9b-hf", - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_gemma7B_tp1_random_128_128", - "server_parameters": { - "model": "google/gemma-7b" - }, - "client_parameters": { - "model": "google/gemma-7b", - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_gemma3-4b_tp1_random_128_128", - "server_environment_variables": { - "VLLM_CPU_SGL_KERNEL": 0 - }, - "server_parameters": { - "model": "google/gemma-3-4b-it" - }, - "client_parameters": { - "model": "google/gemma-3-4b-it", - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_gemma3-12b_tp1_random_128_128", - "server_environment_variables": { - "VLLM_CPU_SGL_KERNEL": 0 - }, - "server_parameters": { - "model": "google/gemma-3-12b-it" - }, - "client_parameters": { - "model": "google/gemma-3-12b-it", - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_gemma4-4b_tp1_random_128_128", - "server_environment_variables": { - "VLLM_CPU_SGL_KERNEL": 0 - }, - "server_parameters": { - "model": "google/gemma-4-E4B-it" - }, - "client_parameters": { - "model": "google/gemma-4-E4B-it", - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_gemma4-2b_tp1_random_128_128", - "server_environment_variables": { - "VLLM_CPU_SGL_KERNEL": 0 - }, - "server_parameters": { - "model": "google/gemma-4-E2B-it" - }, - "client_parameters": { - "model": "google/gemma-4-E2B-it", - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_gemma4-26b_tp1_random_128_128", - "server_environment_variables": { - "VLLM_CPU_SGL_KERNEL": 0, - "VLLM_CPU_ATTN_SPLIT_KV": 0 - }, - "server_parameters": { - "model": "google/gemma-4-26B-A4B-it" - }, - "client_parameters": { - "model": "google/gemma-4-26B-A4B-it", - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_phi4_tp1_random_128_128", - "server_parameters": { - "model": "microsoft/Phi-4-reasoning" - }, - "client_parameters": { - "model": "microsoft/Phi-4-reasoning", - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - } - ] -} diff --git a/.buildkite/performance-benchmarks/tests/serving-tests-cpu.json b/.buildkite/performance-benchmarks/tests/serving-tests-cpu.json deleted file mode 100644 index f1b2f4747a..0000000000 --- a/.buildkite/performance-benchmarks/tests/serving-tests-cpu.json +++ /dev/null @@ -1,141 +0,0 @@ -{ - "defaults": { - "qps_list": [ - "inf" - ], - "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200], - "server_environment_variables": { - "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1, - "VLLM_CPU_SGL_KERNEL": 1, - "VLLM_CPU_KVCACHE_SPACE": 40 - }, - "server_parameters": { - "model": "meta-llama/Llama-3.1-8B-Instruct", - "tensor_parallel_size": 1, - "dtype": "bfloat16", - "distributed_executor_backend": "mp", - "block_size": 128, - "trust_remote_code": "", - "disable_log_stats": "", - "max_num_batched_tokens": 2048, - "max_num_seqs": 256 - }, - "client_parameters": { - "model": "meta-llama/Llama-3.1-8B-Instruct", - "backend": "vllm", - "ignore-eos": "", - "temperature": 0, - "num_prompts": 200 - } - }, - "tests": [ - { - "test_name": "serving_llama8B_tp1_sharegpt", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": { - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json" - } - }, - { - "test_name": "serving_llama8B_tp2_sharegpt", - "server_parameters": { - "tensor_parallel_size": 2 - }, - "client_parameters": { - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json" - } - }, - { - "test_name": "serving_llama8B_tp1_random_128_128", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_llama8B_tp2_random_128_128", - "server_parameters": { - "tensor_parallel_size": 2 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_llama8B_tp1_random_128_2048", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 2048 - } - }, - { - "test_name": "serving_llama8B_tp2_random_128_2048", - "server_parameters": { - "tensor_parallel_size": 2 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 2048 - } - }, - { - "test_name": "serving_llama8B_tp1_random_2048_128", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 2048, - "random-output-len": 128 - } - }, - { - "test_name": "serving_llama8B_tp2_random_2048_128", - "server_parameters": { - "tensor_parallel_size": 2 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 2048, - "random-output-len": 128 - } - }, - { - "test_name": "serving_llama8B_tp1_random_2048_2048", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 2048, - "random-output-len": 2048 - } - }, - { - "test_name": "serving_llama8B_tp2_random_2048_2048", - "server_parameters": { - "tensor_parallel_size": 2 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 2048, - "random-output-len": 2048 - } - } - ] -} diff --git a/.buildkite/performance-benchmarks/tests/serving-tests-hpu.json b/.buildkite/performance-benchmarks/tests/serving-tests-hpu.json deleted file mode 100644 index d5ef981689..0000000000 --- a/.buildkite/performance-benchmarks/tests/serving-tests-hpu.json +++ /dev/null @@ -1,163 +0,0 @@ -[ - { - "test_name": "serving_llama8B_tp1_sharegpt", - "qps_list": [1, 4, 16, "inf"], - "server_environment_variables": { - "PT_HPU_LAZY_MODE": 1, - "VLLM_CONTIGUOUS_PA": 1, - "VLLM_DEFRAG": 1 - }, - "server_parameters": { - "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", - "tensor_parallel_size": 1, - "disable_log_stats": "", - "load_format": "dummy", - "max-model-len": 2048, - "max-num-seqs": 256, - "async-scheduling": "" - }, - "client_parameters": { - "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", - "backend": "vllm", - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "temperature": 0, - "num_prompts": 200 - } - }, - { - "test_name": "serving_llama70B_tp4_sharegpt", - "qps_list": [1, 4, 16, "inf"], - "server_environment_variables": { - "PT_HPU_LAZY_MODE": 1, - "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1, - "VLLM_CONTIGUOUS_PA": 1, - "VLLM_DEFRAG": 1 - }, - "server_parameters": { - "model": "meta-llama/Meta-Llama-3.1-70B-Instruct", - "tensor_parallel_size": 4, - "disable_log_stats": "", - "load_format": "dummy", - "max-model-len": 2048, - "max-num-seqs": 256, - "async-scheduling": "" - }, - "client_parameters": { - "model": "meta-llama/Meta-Llama-3.1-70B-Instruct", - "backend": "vllm", - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "temperature": 0, - "num_prompts": 200 - } - }, - { - "test_name": "serving_mixtral8x7B_tp2_sharegpt", - "qps_list": [1, 4, 16, "inf"], - "server_environment_variables": { - "PT_HPU_LAZY_MODE": 1, - "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1, - "VLLM_CONTIGUOUS_PA": 1, - "VLLM_DEFRAG": 1 - }, - "server_parameters": { - "model": "mistralai/Mixtral-8x7B-Instruct-v0.1", - "tensor_parallel_size": 2, - "disable_log_stats": "", - "load_format": "dummy", - "max-model-len": 2048, - "max-num-seqs": 256, - "async-scheduling": "" - }, - "client_parameters": { - "model": "mistralai/Mixtral-8x7B-Instruct-v0.1", - "backend": "vllm", - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "temperature": 0, - "num_prompts": 200 - } - }, - { - "test_name": "serving_deepseek_r1", - "qps_list": [1, 4, 16, "inf"], - "server_environment_variables": { - "PT_HPU_LAZY_MODE": 1, - "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1, - "VLLM_CONTIGUOUS_PA": 1, - "VLLM_DEFRAG": 1 - }, - "server_parameters": { - "model": "deepseek-ai/DeepSeek-R1", - "tensor_parallel_size": 8, - "disable_log_stats": "", - "load_format": "dummy", - "max-model-len": 2048, - "max-num-seqs": 200, - "async-scheduling": "", - "dtype": "bfloat16" - }, - "client_parameters": { - "model": "deepseek-ai/DeepSeek-R1", - "backend": "vllm", - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "temperature": 0, - "num_prompts": 200 - } - }, - { - "test_name": "serving_llama4_maverick_17b128e_instruct_fp8", - "qps_list": [1, 4, 16, "inf"], - "server_environment_variables": { - "PT_HPU_LAZY_MODE": 1, - "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1, - "VLLM_CONTIGUOUS_PA": 1, - "VLLM_DEFRAG": 1 - }, - "server_parameters": { - "model": "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", - "tensor_parallel_size": 8, - "disable_log_stats": "", - "max-model-len": 2048, - "max-num-seqs": 128, - "async-scheduling": "", - "enable_expert_parallel": "", - "max-num-batched-tokens": 4096 - }, - "client_parameters": { - "model": "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", - "backend": "vllm", - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "temperature": 0, - "num_prompts": 200 - } - }, - { - "test_name": "serving_qwen3_8b", - "qps_list": [1, 4, 10, "inf"], - "server_environment_variables": { - "PT_HPU_LAZY_MODE": 1, - "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1, - "VLLM_CONTIGUOUS_PA": 1, - "VLLM_DEFRAG": 1 - }, - "server_parameters": { - "model": "Qwen/Qwen-3-8B", - "tensor_parallel_size": 1, - "dtype": "bfloat16", - "disable_log_stats": "", - "async-scheduling": "" - }, - "client_parameters": { - "model": "Qwen/Qwen-3-8B", - "backend": "vllm", - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "temperature": 0, - "num_prompts": 200 - } - } -] diff --git a/.buildkite/performance-benchmarks/tests/serving-tests.json b/.buildkite/performance-benchmarks/tests/serving-tests.json deleted file mode 100644 index ac0eb213b9..0000000000 --- a/.buildkite/performance-benchmarks/tests/serving-tests.json +++ /dev/null @@ -1,108 +0,0 @@ -{ - "defaults": { - "qps_list": [ - "inf" - ], - "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200], - "server_parameters": { - "model": "meta-llama/Llama-3.1-8B-Instruct", - "tensor_parallel_size": 1, - "dtype": "bfloat16" - }, - "client_parameters": { - "model": "meta-llama/Llama-3.1-8B-Instruct", - "backend": "vllm", - "ignore-eos": "", - "temperature": 0, - "num_prompts": 200 - } - }, - "tests": [ - { - "test_name": "serving_llama8B_tp1_sharegpt", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": { - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json" - } - }, - { - "test_name": "serving_llama8B_tp1_random_128_128", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_llama8B_tp1_random_128_2048", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 2048 - } - }, - { - "test_name": "serving_llama8B_tp1_random_2048_128", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 2048, - "random-output-len": 128 - } - }, - { - "test_name": "serving_llama8B_tp1_random_2048_2048", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 2048, - "random-output-len": 2048 - } - }, - { - "test_name": "serving_llama70B_tp4_random_128_128", - "server_parameters": { - "model": "meta-llama/Llama-3.3-70B-Instruct", - "tensor_parallel_size": 4, - "async_scheduling": "", - "no_enable_prefix_caching": "", - "max_num_batched_tokens": 8192 - }, - "client_parameters": { - "model": "meta-llama/Llama-3.3-70B-Instruct", - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_gemma4-e4b_tp1_random_128_128", - "server_parameters": { - "model": "google/gemma-4-E4B-it", - "enable_auto_tool_choice": "", - "tool_call_parser": "gemma4", - "chat_template": "examples/tool_chat_template_gemma4.jinja", - "reasoning_parser": "gemma4" - }, - "client_parameters": { - "model": "google/gemma-4-E4B-it", - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - } - ] -} diff --git a/.buildkite/performance-benchmarks/tests/throughput-tests-arm64-cpu.json b/.buildkite/performance-benchmarks/tests/throughput-tests-arm64-cpu.json deleted file mode 100644 index 3bc7dde979..0000000000 --- a/.buildkite/performance-benchmarks/tests/throughput-tests-arm64-cpu.json +++ /dev/null @@ -1,25 +0,0 @@ -[ - { - "test_name": "throughput_llama8B_tp1", - "environment_variables": { - "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1, - "VLLM_CPU_KVCACHE_SPACE": 40 - }, - "parameters": { - "model": "meta-llama/Llama-3.1-8B-Instruct", - "tensor_parallel_size": 1, - "load_format": "dummy", - "dtype": "bfloat16", - "distributed_executor_backend": "mp", - "block_size": 128, - "trust_remote_code": "", - "disable_log_stats": "", - "enforce_eager": "", - "max_num_batched_tokens": 2048, - "max_num_seqs": 256, - "dataset": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "num_prompts": 200, - "backend": "vllm" - } - } -] diff --git a/.buildkite/performance-benchmarks/tests/throughput-tests-cpu.json b/.buildkite/performance-benchmarks/tests/throughput-tests-cpu.json deleted file mode 100644 index 089be00eb4..0000000000 --- a/.buildkite/performance-benchmarks/tests/throughput-tests-cpu.json +++ /dev/null @@ -1,25 +0,0 @@ -[ - { - "test_name": "throughput_llama8B_tp2", - "environment_variables": { - "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1, - "VLLM_CPU_SGL_KERNEL": 1, - "VLLM_CPU_KVCACHE_SPACE": 40 - }, - "parameters": { - "model": "meta-llama/Llama-3.1-8B-Instruct", - "tensor_parallel_size": 2, - "dtype": "bfloat16", - "distributed_executor_backend": "mp", - "block_size": 128, - "trust_remote_code": "", - "disable_log_stats": "", - "enforce_eager": "", - "max_num_batched_tokens": 2048, - "max_num_seqs": 256, - "dataset": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "num_prompts": 200, - "backend": "vllm" - } - } -] diff --git a/.buildkite/performance-benchmarks/tests/throughput-tests-hpu.json b/.buildkite/performance-benchmarks/tests/throughput-tests-hpu.json deleted file mode 100644 index 25344348bb..0000000000 --- a/.buildkite/performance-benchmarks/tests/throughput-tests-hpu.json +++ /dev/null @@ -1,123 +0,0 @@ -[ - { - "test_name": "throughput_llama8B_tp1", - "environment_variables": { - "PT_HPU_LAZY_MODE": 1, - "VLLM_CONTIGUOUS_PA": 1, - "VLLM_DEFRAG": 1 - }, - "parameters": { - "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", - "tensor_parallel_size": 1, - "load_format": "dummy", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "num_prompts": 1000, - "backend": "vllm", - "max-model-len": 2048, - "max-num-seqs": 512, - "async-scheduling": "" - } - }, - { - "test_name": "throughput_llama70B_tp4", - "environment_variables": { - "PT_HPU_LAZY_MODE": 1, - "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1, - "VLLM_CONTIGUOUS_PA": 1, - "VLLM_DEFRAG": 1 - }, - "parameters": { - "model": "meta-llama/Meta-Llama-3.1-70B-Instruct", - "tensor_parallel_size": 4, - "load_format": "dummy", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "num_prompts": 1000, - "backend": "vllm", - "max-model-len": 2048, - "max-num-seqs": 512, - "async-scheduling": "" - } - }, - { - "test_name": "throughput_mixtral8x7B_tp2", - "environment_variables": { - "PT_HPU_LAZY_MODE": 1, - "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1, - "VLLM_CONTIGUOUS_PA": 1, - "VLLM_DEFRAG": 1 - }, - "parameters": { - "model": "mistralai/Mixtral-8x7B-Instruct-v0.1", - "tensor_parallel_size": 2, - "load_format": "dummy", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "num_prompts": 1000, - "backend": "vllm", - "max-model-len": 2048, - "max-num-seqs": 512, - "async-scheduling": "" - } - }, - { - "test_name": "throughput_deepseek_r1", - "environment_variables": { - "PT_HPU_LAZY_MODE": 1, - "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1, - "VLLM_CONTIGUOUS_PA": 1, - "VLLM_DEFRAG": 1 - }, - "parameters": { - "model": "deepseek-ai/DeepSeek-R1", - "tensor_parallel_size": 8, - "load_format": "dummy", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "dataset_name": "sharegpt", - "num_prompts": 1000, - "backend": "vllm", - "max-model-len": 2048, - "max-num-seqs": 384, - "async-scheduling": "" - } - }, - { - "test_name": "throughput_llama4_maverick_17b128e_instruct_fp8", - "environment_variables": { - "PT_HPU_LAZY_MODE": 1, - "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1, - "VLLM_CONTIGUOUS_PA": 1, - "VLLM_DEFRAG": 1 - }, - "parameters": { - "model": "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", - "tensor_parallel_size": 8, - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "dataset_name": "sharegpt", - "num_prompts": 1000, - "backend": "vllm", - "max-model-len": 2048, - "max-num-seqs": 512, - "async-scheduling": "", - "enable_expert_parallel": "" - } - }, - { - "test_name": "throughput_qwen3_8b", - "environment_variables": { - "PT_HPU_LAZY_MODE": 1, - "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1, - "VLLM_CONTIGUOUS_PA": 1, - "VLLM_DEFRAG": 1 - }, - "parameters": { - "model": "Qwen/Qwen-3-8B", - "tensor_parallel_size": 1, - "load_format": "dummy", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "dataset_name": "sharegpt", - "num_prompts": 1000, - "max-num-seqs": 512, - "backend": "vllm", - "async-scheduling": "" - } - } -] diff --git a/.buildkite/performance-benchmarks/tests/throughput-tests.json b/.buildkite/performance-benchmarks/tests/throughput-tests.json deleted file mode 100644 index 9bc87cbcd2..0000000000 --- a/.buildkite/performance-benchmarks/tests/throughput-tests.json +++ /dev/null @@ -1,35 +0,0 @@ -[ - { - "test_name": "throughput_llama8B_tp1", - "parameters": { - "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", - "tensor_parallel_size": 1, - "load_format": "dummy", - "dataset": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "num_prompts": 200, - "backend": "vllm" - } - }, - { - "test_name": "throughput_llama70B_tp4", - "parameters": { - "model": "meta-llama/Meta-Llama-3.1-70B-Instruct", - "tensor_parallel_size": 4, - "load_format": "dummy", - "dataset": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "num_prompts": 200, - "backend": "vllm" - } - }, - { - "test_name": "throughput_mixtral8x7B_tp2", - "parameters": { - "model": "mistralai/Mixtral-8x7B-Instruct-v0.1", - "tensor_parallel_size": 2, - "load_format": "dummy", - "dataset": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "num_prompts": 200, - "backend": "vllm" - } - } -] diff --git a/.buildkite/release-pipeline.yaml b/.buildkite/release-pipeline.yaml deleted file mode 100644 index 84987524b2..0000000000 --- a/.buildkite/release-pipeline.yaml +++ /dev/null @@ -1,1055 +0,0 @@ -# CUDA architecture lists — following PyTorch RELEASE.md -# (https://github.com/pytorch/pytorch/blob/main/RELEASE.md) -# SM86 included for broader Ampere coverage; SM89 for marlin fp8 support -# These requested arches are filtered by CMake's CUDA_SUPPORTED_ARCHS before -# per-kernel arch selection. Do not add +PTX here: top-level +PTX is stripped -# during that filtering, so kernels that need PTX must request it locally. -env: - # for CUDA >=13, sm_100+ targets have family specifiers (see CMakeLists.txt) - # so targets like 10.3 and 12.1 are automatically supported with this list - CUDA_ARCH_X86: "7.5 8.0 8.6 8.9 9.0 10.0 12.0" - # aarch64-only targets: Orin (8.7), Thor (11.0, CUDA 13+) - CUDA_ARCH_AARCH64: "8.0 8.7 8.9 9.0 10.0 11.0 12.0" - - # for CUDA <13, we need to specify all needed targets - # some targets (10.3, 12.1) are skipped to limit the wheel size (< 500MB) - # please use CUDA 13 wheels or compile yourself on these new devices - CUDA_ARCH_X86_CU129: "7.5 8.0 8.6 8.9 9.0 10.0 12.0" - CUDA_ARCH_AARCH64_CU129: "8.0 8.7 8.9 9.0 10.0 12.0" - -steps: - - input: "Provide Release version here" - id: input-release-version - fields: - - text: "What is the release version?" - key: release-version - - - group: "Build CUDA 13.0 Python wheels" - key: "build-wheels" - steps: - - label: "Build wheel - aarch64 - CUDA 13.0" - depends_on: ~ - id: build-wheel-arm64-cuda-13-0 - agents: - queue: arm64_cpu_queue_release - commands: - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.0-b8b5f17a7d9ccfc25bbc5cf17b3fcea12964a042 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." - - "mkdir artifacts" - - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - - "bash .buildkite/scripts/upload-nightly-wheels.sh" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)" release-wheels' - env: - DOCKER_BUILDKIT: "1" - - - label: "Build wheel - x86_64 - CUDA 13.0" - depends_on: ~ - id: build-wheel-x86-cuda-13-0 - agents: - queue: cpu_queue_release - commands: - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda13.0-78e737ad29420ffc4800e677c51e2a852caf8359 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." - - "mkdir artifacts" - - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - - "bash .buildkite/scripts/upload-nightly-wheels.sh" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)" release-wheels' - env: - DOCKER_BUILDKIT: "1" - - - block: "Unblock to build additional Python wheels" - depends_on: ~ - key: block-build-additional-wheels - if: build.env("NIGHTLY") != "1" - - - group: "Build additional Python wheels" - key: "build-additional-wheels" - depends_on: block-build-additional-wheels - allow_dependency_failure: true - steps: - - label: "Build wheel - aarch64 - CUDA 12.9" - depends_on: ~ - id: build-wheel-arm64-cuda-12-9 - agents: - queue: arm64_cpu_queue_release - commands: - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64_CU129}\" --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda12.9-78e737ad29420ffc4800e677c51e2a852caf8359 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." - - "mkdir artifacts" - - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - - "bash .buildkite/scripts/upload-nightly-wheels.sh" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)" release-wheels' - env: - DOCKER_BUILDKIT: "1" - - - label: "Build wheel - aarch64 - CPU" - depends_on: ~ - id: build-wheel-arm64-cpu - agents: - queue: arm64_cpu_queue_release - commands: - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --build-arg VLLM_BUILD_ACL=ON --tag vllm-ci:build-image --target vllm-build --progress plain -f docker/Dockerfile.cpu ." - - "mkdir artifacts" - - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - - "bash .buildkite/scripts/upload-nightly-wheels.sh" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)" release-wheels' - env: - DOCKER_BUILDKIT: "1" - - - label: "Build wheel - macOS arm64 - CPU" - depends_on: ~ - id: build-wheel-macos-arm64-cpu - agents: - queue: macmini - timeout_in_minutes: 90 - commands: - - "bash .buildkite/scripts/build-macos-wheel.sh" - - "mkdir -p artifacts/transfer" - - "basename artifacts/dist/*.whl > artifacts/transfer/wheel.name" - - "shasum -a 256 artifacts/dist/*.whl > artifacts/transfer/wheel.sha256" - - "split -b 1m artifacts/dist/*.whl artifacts/transfer/wheel.part." - env: - BUILDKITE_ARTIFACT_UPLOAD_CONCURRENCY: "1" - BUILDKITE_NO_HTTP2: "true" - artifact_paths: - - "artifacts/transfer/*" - - - label: "Upload wheel - macOS arm64 - CPU" - depends_on: build-wheel-macos-arm64-cpu - id: upload-wheel-macos-arm64-cpu - agents: - queue: small_cpu_queue_release - timeout_in_minutes: 15 - commands: - - 'buildkite-agent artifact download "artifacts/transfer/*" .' - - "mkdir -p artifacts/dist artifacts/reassembled" - - 'wheel_name=$$(cat artifacts/transfer/wheel.name)' - - '[[ "$$wheel_name" =~ ^[A-Za-z0-9][A-Za-z0-9._+-]*\.whl$$ && "$$wheel_name" != *..* ]]' - - 'expected_sha=$$(awk ''{print $$1}'' artifacts/transfer/wheel.sha256)' - - '[[ "$$expected_sha" =~ ^[0-9a-f]{64}$$ ]]' - - 'cat artifacts/transfer/wheel.part.* > artifacts/reassembled/wheel' - - 'actual_sha=$$(shasum -a 256 artifacts/reassembled/wheel | awk ''{print $$1}'')' - - '[[ "$$actual_sha" == "$$expected_sha" ]]' - - 'mv artifacts/reassembled/wheel "artifacts/dist/$$wheel_name"' - - "aws sts get-caller-identity" - - "VLLM_WHEEL_PLATFORM=macos bash .buildkite/scripts/upload-nightly-wheels.sh" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)" release-wheels' - plugins: - - aws-assume-role-with-web-identity#v1.6.0: - role-arn: arn:aws:iam::936637512419:role/vllm-release-macos-wheel-uploader - region: us-west-2 - - - label: "Build wheel - x86_64 - CUDA 12.9" - depends_on: ~ - id: build-wheel-x86-cuda-12-9 - agents: - queue: cpu_queue_release - commands: - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86_CU129}\" --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda12.9-78e737ad29420ffc4800e677c51e2a852caf8359 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." - - "mkdir artifacts" - - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - - "bash .buildkite/scripts/upload-nightly-wheels.sh" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)" release-wheels' - env: - DOCKER_BUILDKIT: "1" - - - label: "Build wheel - x86_64 - CPU" - depends_on: ~ - id: build-wheel-x86-cpu - agents: - queue: cpu_queue_release - commands: - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --build-arg VLLM_CPU_X86=true --tag vllm-ci:build-image --target vllm-build --progress plain -f docker/Dockerfile.cpu ." - - "mkdir artifacts" - - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - - "bash .buildkite/scripts/upload-nightly-wheels.sh" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)" release-wheels' - env: - DOCKER_BUILDKIT: "1" - - - label: "Build wheel - x86_64 - XPU" - depends_on: ~ - id: build-wheel-x86-xpu - agents: - queue: cpu_queue_release - commands: - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --tag vllm-ci:build-image --target vllm-build --progress plain -f docker/Dockerfile.xpu ." - - "mkdir artifacts" - - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - - "bash .buildkite/scripts/upload-nightly-wheels.sh" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)" release-wheels' - env: - DOCKER_BUILDKIT: "1" - - - label: "Publish and stage XPU Triton shim" - key: publish-xpu-triton-shim - depends_on: ~ - agents: - queue: cpu_queue_release - commands: - - "bash .buildkite/scripts/xpu/publish-triton-shim.sh" - - - label: "Generate and upload wheel indices" - key: generate-wheel-indices - depends_on: "build-wheels" - allow_dependency_failure: true - if: build.env("NIGHTLY") != "1" - agents: - queue: cpu_queue_release - commands: - - "UPDATE_VERSION_INDEX=0 bash .buildkite/scripts/generate-and-upload-nightly-index.sh" - - - label: "Regenerate indices with additional wheels" - key: generate-additional-wheel-indices - depends_on: - - build-wheels - - build-additional-wheels - - generate-wheel-indices - allow_dependency_failure: true - agents: - queue: cpu_queue_release - commands: - - 'UPDATE_NIGHTLY_INDEX="$${NIGHTLY:-0}" bash .buildkite/scripts/generate-and-upload-nightly-index.sh' - - - block: "Unblock to build release Docker images" - depends_on: ~ - key: block-build-release-images - if: build.env("NIGHTLY") != "1" - - - group: "Build release Docker images" - key: "build-release-images" - depends_on: block-build-release-images - allow_dependency_failure: true - steps: - - label: "Build release image - x86_64 - CUDA 13.0" - depends_on: ~ - id: build-release-image-x86 - agents: - queue: cpu_queue_release - commands: - - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - | - DOCKER_BUILDKIT=1 docker build \ - $(bash .buildkite/scripts/docker-build-metadata-args.sh) \ - --build-arg max_jobs=16 \ - --build-arg USE_SCCACHE=1 \ - --build-arg GIT_REPO_CHECK=1 \ - --build-arg CUDA_VERSION=13.0.2 \ - --build-arg torch_cuda_arch_list="${CUDA_ARCH_X86}" \ - --build-arg INSTALL_KV_CONNECTORS=true \ - --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda13.0-78e737ad29420ffc4800e677c51e2a852caf8359 \ - --target vllm-openai \ - --progress plain \ - -f docker/Dockerfile . - - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)" - # re-tag to default image tag and push, just in case arm64 build fails - - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT" - - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)" release-images' - - - label: "Build release image - aarch64 - CUDA 13.0" - depends_on: ~ - id: build-release-image-arm64 - agents: - queue: arm64_cpu_queue_release - commands: - - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - | - DOCKER_BUILDKIT=1 docker build \ - $(bash .buildkite/scripts/docker-build-metadata-args.sh) \ - --build-arg max_jobs=16 \ - --build-arg USE_SCCACHE=1 \ - --build-arg GIT_REPO_CHECK=1 \ - --build-arg CUDA_VERSION=13.0.2 \ - --build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64}" \ - --build-arg INSTALL_KV_CONNECTORS=true \ - --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.0-b8b5f17a7d9ccfc25bbc5cf17b3fcea12964a042 \ - --target vllm-openai \ - --progress plain \ - -f docker/Dockerfile . - - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)" release-images' - - - label: "Build release image - x86_64 - CUDA 12.9" - depends_on: ~ - id: build-release-image-x86-cuda-12-9 - agents: - queue: cpu_queue_release - commands: - - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - | - DOCKER_BUILDKIT=1 docker build \ - $(bash .buildkite/scripts/docker-build-metadata-args.sh cu129) \ - --build-arg max_jobs=16 \ - --build-arg USE_SCCACHE=1 \ - --build-arg GIT_REPO_CHECK=1 \ - --build-arg CUDA_VERSION=12.9.1 \ - --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda12.9-78e737ad29420ffc4800e677c51e2a852caf8359 \ - --build-arg torch_cuda_arch_list="${CUDA_ARCH_X86_CU129}" \ - --build-arg INSTALL_KV_CONNECTORS=true \ - --target vllm-openai \ - --progress plain \ - -f docker/Dockerfile . - - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129" - # re-tag to default image tag and push, just in case arm64 build fails - - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129" - - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129" release-images' - - - label: "Build release image - aarch64 - CUDA 12.9" - depends_on: ~ - id: build-release-image-arm64-cuda-12-9 - agents: - queue: arm64_cpu_queue_release - commands: - - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - | - DOCKER_BUILDKIT=1 docker build \ - $(bash .buildkite/scripts/docker-build-metadata-args.sh cu129) \ - --build-arg max_jobs=16 \ - --build-arg USE_SCCACHE=1 \ - --build-arg GIT_REPO_CHECK=1 \ - --build-arg CUDA_VERSION=12.9.1 \ - --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda12.9-78e737ad29420ffc4800e677c51e2a852caf8359 \ - --build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64_CU129}" \ - --build-arg INSTALL_KV_CONNECTORS=true \ - --target vllm-openai \ - --progress plain \ - -f docker/Dockerfile . - - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129" release-images' - - - label: "Build release image - x86_64 - Rubin CUDA 13.4" - depends_on: ~ - id: build-release-image-x86-rubin-cuda-13-4 - if: build.env("NIGHTLY") == "1" - agents: - queue: cpu_queue_release - commands: - - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - | - DOCKER_BUILDKIT=1 docker buildx build --load \ - $(bash .buildkite/scripts/docker-build-metadata-args.sh cu134) \ - --platform linux/amd64 \ - --build-arg max_jobs=16 \ - --build-arg USE_SCCACHE=1 \ - --build-arg GIT_REPO_CHECK=1 \ - --build-arg CUDA_VERSION=13.4 \ - --build-arg torch_cuda_arch_list="${CUDA_ARCH_X86}" \ - --build-arg INSTALL_KV_CONNECTORS=true \ - --build-arg INSTALL_RUBIN_PRERELEASE=true \ - --build-arg TRITON_INSTALL_FROM_SOURCE_REPO=https://github.com/triton-lang/triton.git \ - --build-arg TRITON_INSTALL_FROM_SOURCE_REVISION=3f6e41132b5edf639bfb872ad73d4688765e08b8 \ - --build-arg RUN_WHEEL_CHECK=false \ - --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda13.4 \ - --build-arg FINAL_BASE_IMAGE=nvidia/cuda:13.4.1-base-ubuntu24.04 \ - --build-arg NCCL_VERSION=2.32.3 \ - --target vllm-openai \ - --progress plain \ - -f docker/Dockerfile . - - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134" release-images' - - - label: "Build release image - aarch64 - Rubin CUDA 13.4" - depends_on: ~ - id: build-release-image-arm64-rubin-cuda-13-4 - if: build.env("NIGHTLY") == "1" - agents: - queue: arm64_cpu_queue_release - commands: - - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - # TODO(arpera): - # Remove BUILD_NIXL=true once NIXL has a wheel built against torch 2.15 - # For details: https://github.com/vllm-project/vllm/pull/60090 - # Tracking issue report: https://github.com/ai-dynamo/nixl/issues/2297 - - | - DOCKER_BUILDKIT=1 docker buildx build --load \ - $(bash .buildkite/scripts/docker-build-metadata-args.sh cu134) \ - --platform linux/arm64 \ - --build-arg max_jobs=16 \ - --build-arg USE_SCCACHE=1 \ - --build-arg GIT_REPO_CHECK=1 \ - --build-arg CUDA_VERSION=13.4 \ - --build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64}" \ - --build-arg INSTALL_KV_CONNECTORS=true \ - --build-arg INSTALL_RUBIN_PRERELEASE=true \ - --build-arg BUILD_NIXL=true \ - --build-arg TRITON_INSTALL_FROM_SOURCE_REPO=https://github.com/triton-lang/triton.git \ - --build-arg TRITON_INSTALL_FROM_SOURCE_REVISION=3f6e41132b5edf639bfb872ad73d4688765e08b8 \ - --build-arg RUN_WHEEL_CHECK=false \ - --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.4 \ - --build-arg FINAL_BASE_IMAGE=nvidia/cuda:13.4.1-base-ubuntu24.04 \ - --build-arg NCCL_VERSION=2.32.3 \ - --target vllm-openai \ - --progress plain \ - -f docker/Dockerfile . - - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134" release-images' - - - label: "Build release image - x86_64 - CUDA 13.0 - Ubuntu 24.04" - depends_on: ~ - id: build-release-image-x86-ubuntu2404 - agents: - queue: cpu_queue_release - commands: - - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - | - DOCKER_BUILDKIT=1 docker build \ - $(bash .buildkite/scripts/docker-build-metadata-args.sh ubuntu2404) \ - --build-arg max_jobs=16 \ - --build-arg USE_SCCACHE=1 \ - --build-arg GIT_REPO_CHECK=1 \ - --build-arg CUDA_VERSION=13.0.2 \ - --build-arg UBUNTU_VERSION=24.04 \ - --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 \ - --build-arg torch_cuda_arch_list="${CUDA_ARCH_X86}" \ - --build-arg INSTALL_KV_CONNECTORS=true \ - --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda13.0-78e737ad29420ffc4800e677c51e2a852caf8359 \ - --target vllm-openai \ - --progress plain \ - -f docker/Dockerfile . - - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404" - - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404" - - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404" release-images' - - - label: "Build release image - aarch64 - CUDA 13.0 - Ubuntu 24.04" - depends_on: ~ - id: build-release-image-arm64-ubuntu2404 - agents: - queue: arm64_cpu_queue_release - commands: - - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - | - DOCKER_BUILDKIT=1 docker build \ - $(bash .buildkite/scripts/docker-build-metadata-args.sh ubuntu2404) \ - --build-arg max_jobs=16 \ - --build-arg USE_SCCACHE=1 \ - --build-arg GIT_REPO_CHECK=1 \ - --build-arg CUDA_VERSION=13.0.2 \ - --build-arg UBUNTU_VERSION=24.04 \ - --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 \ - --build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64}" \ - --build-arg INSTALL_KV_CONNECTORS=true \ - --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.0-b8b5f17a7d9ccfc25bbc5cf17b3fcea12964a042 \ - --target vllm-openai \ - --progress plain \ - -f docker/Dockerfile . - - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404" release-images' - - - label: "Build release image - x86_64 - CUDA 12.9 - Ubuntu 24.04" - depends_on: ~ - id: build-release-image-x86-cuda-12-9-ubuntu2404 - agents: - queue: cpu_queue_release - commands: - - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - | - DOCKER_BUILDKIT=1 docker build \ - $(bash .buildkite/scripts/docker-build-metadata-args.sh cu129-ubuntu2404) \ - --build-arg max_jobs=16 \ - --build-arg USE_SCCACHE=1 \ - --build-arg GIT_REPO_CHECK=1 \ - --build-arg CUDA_VERSION=12.9.1 \ - --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda12.9-78e737ad29420ffc4800e677c51e2a852caf8359 \ - --build-arg UBUNTU_VERSION=24.04 \ - --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 \ - --build-arg torch_cuda_arch_list="${CUDA_ARCH_X86_CU129}" \ - --build-arg INSTALL_KV_CONNECTORS=true \ - --target vllm-openai \ - --progress plain \ - -f docker/Dockerfile . - - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404" - - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404" - - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404" release-images' - - - label: "Build release image - aarch64 - CUDA 12.9 - Ubuntu 24.04" - depends_on: ~ - id: build-release-image-arm64-cuda-12-9-ubuntu2404 - agents: - queue: arm64_cpu_queue_release - commands: - - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - | - DOCKER_BUILDKIT=1 docker build \ - $(bash .buildkite/scripts/docker-build-metadata-args.sh cu129-ubuntu2404) \ - --build-arg max_jobs=16 \ - --build-arg USE_SCCACHE=1 \ - --build-arg GIT_REPO_CHECK=1 \ - --build-arg CUDA_VERSION=12.9.1 \ - --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda12.9-78e737ad29420ffc4800e677c51e2a852caf8359 \ - --build-arg UBUNTU_VERSION=24.04 \ - --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 \ - --build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64_CU129}" \ - --build-arg INSTALL_KV_CONNECTORS=true \ - --target vllm-openai \ - --progress plain \ - -f docker/Dockerfile . - - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404" release-images' - - - label: ":docker: Build release image - x86_64 - XPU" - depends_on: ~ - id: build-xpu-release-image - agents: - queue: cpu_queue_release - commands: - - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - | - DOCKER_BUILDKIT=1 docker build \ - $(bash .buildkite/scripts/docker-build-metadata-args.sh xpu) \ - --build-arg GIT_REPO_CHECK=1 \ - --target vllm-openai \ - --progress plain \ - -f docker/Dockerfile.xpu . - - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-xpu" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-xpu" release-images' - env: - DOCKER_BUILDKIT: "1" - - # CPU release images follow the CUDA pattern: a single build label per arch - # pushes the arch-specific commit tag to ECR, consumed by both the nightly - # DockerHub publish and the release publish step. The block auto-skips in - # nightly (if NIGHTLY != "1") so the build runs unattended, mirroring - # block-build-release-images. - - block: "Build release image for x86_64 CPU" - key: block-cpu-release-image-build - depends_on: ~ - if: build.env("NIGHTLY") != "1" - - - label: "Build release image - x86_64 - CPU" - key: build-cpu-release-image-x86 - depends_on: - - block-cpu-release-image-build - allow_dependency_failure: true - agents: - queue: cpu_queue_release - commands: - - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --build-arg VLLM_CPU_X86=true --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$BUILDKITE_COMMIT-x86_64 --progress plain --target vllm-openai -f docker/Dockerfile.cpu ." - - "docker push public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$BUILDKITE_COMMIT-x86_64" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$BUILDKITE_COMMIT-x86_64" release-images' - env: - DOCKER_BUILDKIT: "1" - - - block: "Build release image for arm64 CPU" - key: block-arm64-cpu-release-image-build - depends_on: ~ - if: build.env("NIGHTLY") != "1" - - - label: "Build release image - arm64 - CPU" - key: build-cpu-release-image-arm64 - depends_on: - - block-arm64-cpu-release-image-build - allow_dependency_failure: true - agents: - queue: arm64_cpu_queue_release - commands: - - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --tag public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:$BUILDKITE_COMMIT-arm64 --progress plain --target vllm-openai -f docker/Dockerfile.cpu ." - - "docker push public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:$BUILDKITE_COMMIT-arm64" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:$BUILDKITE_COMMIT-arm64" release-images' - env: - DOCKER_BUILDKIT: "1" - - - group: "Publish release images" - key: "publish-release-images" - steps: - - label: "Create multi-arch manifest - CUDA 13.0" - depends_on: - - build-release-image-x86 - - build-release-image-arm64 - id: create-multi-arch-manifest - agents: - queue: small_cpu_queue_release - commands: - - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64 --amend" - - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: CUDA 13.0" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT" release-manifests' - - - label: "Create multi-arch manifest - CUDA 12.9" - depends_on: - - build-release-image-x86-cuda-12-9 - - build-release-image-arm64-cuda-12-9 - id: create-multi-arch-manifest-cuda-12-9 - agents: - queue: small_cpu_queue_release - commands: - - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu129 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu129 --amend" - - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: CUDA 12.9" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129" release-manifests' - - - label: "Create multi-arch manifest - Rubin CUDA 13.4" - depends_on: - - build-release-image-x86-rubin-cuda-13-4 - - build-release-image-arm64-rubin-cuda-13-4 - id: create-multi-arch-manifest-rubin-cuda-13-4 - if: build.env("NIGHTLY") == "1" - agents: - queue: small_cpu_queue_release - commands: - - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu134 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu134 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu134 --amend" - - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu134" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: Rubin CUDA 13.4" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu134" release-manifests' - - - label: "Create multi-arch manifest - CUDA 13.0 - Ubuntu 24.04" - depends_on: - - build-release-image-x86-ubuntu2404 - - build-release-image-arm64-ubuntu2404 - id: create-multi-arch-manifest-ubuntu2404 - agents: - queue: small_cpu_queue_release - commands: - - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-ubuntu2404 --amend" - - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: CUDA 13.0 Ubuntu 24.04" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404" release-manifests' - - - label: "Create multi-arch manifest - CUDA 12.9 - Ubuntu 24.04" - depends_on: - - build-release-image-x86-cuda-12-9-ubuntu2404 - - build-release-image-arm64-cuda-12-9-ubuntu2404 - id: create-multi-arch-manifest-cuda-12-9-ubuntu2404 - agents: - queue: small_cpu_queue_release - commands: - - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu129-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu129-ubuntu2404 --amend" - - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: CUDA 12.9 Ubuntu 24.04" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404" release-manifests' - - - label: "Create manifest - XPU" - depends_on: - - build-xpu-release-image - id: create-manifest-xpu - agents: - queue: small_cpu_queue_release - commands: - - "bash .buildkite/scripts/xpu/create-xpu-ecr-manifest.sh" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: XPU" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-xpu" release-manifests' - - - label: "Publish nightly multi-arch image to DockerHub" - depends_on: - - create-multi-arch-manifest - if: build.env("NIGHTLY") == "1" - agents: - queue: small_cpu_queue_release - commands: - - "bash .buildkite/scripts/push-nightly-builds.sh" - # Clean up old nightly builds (keep only last 14) - - "bash .buildkite/scripts/cleanup-nightly-builds.sh" - plugins: - - docker-login#v3.0.0: - username: vllmbot - password-env: DOCKERHUB_TOKEN - env: - DOCKER_BUILDKIT: "1" - DOCKERHUB_USERNAME: "vllmbot" - - - label: "Publish nightly multi-arch image to DockerHub - CUDA 12.9" - depends_on: - - create-multi-arch-manifest-cuda-12-9 - if: build.env("NIGHTLY") == "1" - agents: - queue: small_cpu_queue_release - commands: - - "bash .buildkite/scripts/push-nightly-builds.sh cu129" - # Clean up old nightly builds (keep only last 14) - - "bash .buildkite/scripts/cleanup-nightly-builds.sh cu129-nightly-" - plugins: - - docker-login#v3.0.0: - username: vllmbot - password-env: DOCKERHUB_TOKEN - env: - DOCKER_BUILDKIT: "1" - DOCKERHUB_USERNAME: "vllmbot" - - - label: "Publish nightly multi-arch Rubin image to DockerHub - CUDA 13.4" - depends_on: - - create-multi-arch-manifest-rubin-cuda-13-4 - if: build.env("NIGHTLY") == "1" - agents: - queue: small_cpu_queue_release - commands: - - "bash .buildkite/scripts/push-nightly-builds.sh cu134" - # Clean up old nightly builds (keep only last 14) - - "bash .buildkite/scripts/cleanup-nightly-builds.sh cu134-nightly-" - plugins: - - docker-login#v3.0.0: - username: vllmbot - password-env: DOCKERHUB_TOKEN - env: - DOCKER_BUILDKIT: "1" - DOCKERHUB_USERNAME: "vllmbot" - - - label: "Publish nightly CPU multi-arch image to DockerHub" - depends_on: - - build-cpu-release-image-x86 - - build-cpu-release-image-arm64 - if: build.env("NIGHTLY") == "1" - agents: - queue: small_cpu_queue_release - commands: - - "bash .buildkite/scripts/push-nightly-builds-cpu.sh" - # Clean up old nightly builds (keep only last 14) - - "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly- vllm/vllm-openai-cpu" - plugins: - - docker-login#v3.0.0: - username: vllmbot - password-env: DOCKERHUB_TOKEN - env: - DOCKER_BUILDKIT: "1" - DOCKERHUB_USERNAME: "vllmbot" - - - trigger: "perf-eval" - label: "Trigger perf-eval (CUDA)" - key: trigger-perf-eval-cuda - if: build.env("NIGHTLY") == "1" || build.env("RUN_PERF_EVAL") == "1" - depends_on: - - build-release-image-x86 - async: true - soft_fail: true - build: - branch: "main" - message: "(NIGHTLY=${NIGHTLY:-0}) Evaluate vLLM $BUILDKITE_COMMIT (CUDA)" - env: - NIGHTLY: 1 # run all perf-eval workloads - VLLM_COMMIT: "$BUILDKITE_COMMIT" - VLLM_IMAGE_CUDA: "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64" - - - trigger: "perf-eval" - label: "Trigger perf-eval (ROCm)" - key: trigger-perf-eval-rocm - if: build.env("NIGHTLY") == "1" || build.env("RUN_PERF_EVAL") == "1" - depends_on: - - build-rocm100-release-image - async: true - soft_fail: true - build: - branch: "main" - message: "(NIGHTLY=${NIGHTLY:-0}) Evaluate vLLM $BUILDKITE_COMMIT (ROCm)" - env: - NIGHTLY: 1 # run all perf-eval workloads - VLLM_COMMIT: "$BUILDKITE_COMMIT" - VLLM_IMAGE_ROCM: "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-rocm100" - - # ============================================================================= - # ROCm Release Pipelines (x86_64 only) - # ============================================================================= - # - # One group per ROCm stack, identified by its base Dockerfile (ROCM_BASE_DOCKERFILE). - # The scripts derive the ROCm version, wheel variant, cache keys and image tags from - # it (.buildkite/scripts/rocm/stack.sh), so both groups share one set of step - # definitions, anchored on the ROCm 10.0 group. - # - # - ROCm 10.0 (TheRock, docker/Dockerfile.rocm{,_base}) is the default: it owns the - # :nightly / :latest images and the root wheel index. - # - ROCm 7.2 (docker/Dockerfile.rocm_72{,_base}) is kept for a transition period as - # rocm72 images and wheels. - # - # vLLM version is determined by the Buildkite checkout (like the CUDA pipeline). - # ============================================================================= - - - group: "Build ROCm 10.0 Wheel / Image" - key: "build-rocm100" - depends_on: ~ - steps: - - label: ":rocm: Build ROCm 10.0 Base Image & Wheels" - id: build-rocm100-base - depends_on: ~ - agents: &rocm_build_agents - queue: cpu_queue_release - timeout_in_minutes: 360 - env: &rocm100_env - ROCM_BASE_DOCKERFILE: "docker/Dockerfile.rocm_base" - ROCM_DEFAULT_ARG: "--default" - DOCKER_BUILDKIT: "1" - S3_BUCKET: "vllm-wheels" - DOCKERHUB_USERNAME: "vllmbot" - commands: &rocm_base_commands - - | - set -euo pipefail - source .buildkite/scripts/rocm/stack.sh "$${ROCM_BASE_DOCKERFILE}" - SCCACHE_ARGS=(--build-arg USE_SCCACHE=1 --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache - --build-arg SCCACHE_REGION_NAME=us-west-2 --build-arg SCCACHE_S3_NO_CREDENTIALS=0) - aws ecr-public get-login-password --region us-east-1 | \ - docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7 - - # Base image, cached in ECR by the base Dockerfile's hash - if docker manifest inspect "$${ROCM_STACK_ECR_BASE}" > /dev/null 2>&1; then - echo "ECR image cache HIT: $${ROCM_STACK_ECR_BASE}" - else - docker buildx build --file "$${ROCM_BASE_DOCKERFILE}" --tag "$${ROCM_STACK_ECR_BASE}" \ - "$${SCCACHE_ARGS[@]}" --push . - fi - - # Base wheels, cached in S3 (see cache-rocm-base-wheels.sh) - CACHE=.buildkite/scripts/cache-rocm-base-wheels.sh - if [ "$$($${CACHE} "$${ROCM_BASE_DOCKERFILE}" check)" = "hit" ]; then - $${CACHE} "$${ROCM_BASE_DOCKERFILE}" download - else - DEBS="rocm-base-debs:$${BUILDKITE_BUILD_NUMBER}-$${ROCM_STACK_VARIANT}" - docker buildx build --file "$${ROCM_BASE_DOCKERFILE}" --tag "$${DEBS}" \ - --target debs_wheel_release "$${SCCACHE_ARGS[@]}" --load . - mkdir -p "$${ROCM_STACK_BASE_WHEELS_DIR}" - cid=$$(docker create "$${DEBS}") - docker cp "$${cid}":/app/debs/. "$${ROCM_STACK_BASE_WHEELS_DIR}/" - docker rm "$${cid}" - $${CACHE} "$${ROCM_BASE_DOCKERFILE}" upload - fi - buildkite-agent artifact upload "$${ROCM_STACK_BASE_WHEELS_DIR}/*.whl" - - - label: ":python: Build vLLM ROCm 10.0 Wheel - x86_64" - id: build-rocm100-vllm-wheel - depends_on: build-rocm100-base - agents: *rocm_build_agents - timeout_in_minutes: 180 - env: *rocm100_env - commands: &rocm_vllm_wheel_commands - - | - set -euo pipefail - source .buildkite/scripts/rocm/stack.sh "$${ROCM_BASE_DOCKERFILE}" - # Buildkite's default fetch doesn't update moved tags; version detection needs them - git fetch --tags --force origin - buildkite-agent artifact download "$${ROCM_STACK_BASE_WHEELS_DIR}/*.whl" . - aws ecr-public get-login-password --region us-east-1 | \ - docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7 - docker pull "$${ROCM_STACK_ECR_BASE}" - mkdir -p docker/context/base-wheels - touch docker/context/base-wheels/.keep - cp "$${ROCM_STACK_BASE_WHEELS_DIR}"/*.whl docker/context/base-wheels/ - docker build --file "$${ROCM_STACK_DOCKERFILE}" --target export_vllm_wheel_release \ - --output type=local,dest=rocm-dist \ - --build-arg BASE_IMAGE="$${ROCM_STACK_ECR_BASE}" --build-arg REMOTE_VLLM=0 \ - --build-arg GIT_REPO_CHECK=1 --build-arg USE_SCCACHE=1 \ - --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \ - --build-arg SCCACHE_REGION_NAME=us-west-2 --build-arg SCCACHE_S3_NO_CREDENTIALS=0 . - mkdir -p "$${ROCM_STACK_VLLM_WHEEL_DIR}" - cp rocm-dist/*.whl "$${ROCM_STACK_VLLM_WHEEL_DIR}/" - buildkite-agent artifact upload "$${ROCM_STACK_VLLM_WHEEL_DIR}/*.whl" - - - label: ":s3: Upload ROCm 10.0 Wheels to S3" - id: upload-rocm100-wheels - depends_on: build-rocm100-vllm-wheel - agents: *rocm_build_agents - timeout_in_minutes: 60 - env: *rocm100_env - commands: &rocm_upload_commands - - | - set -euo pipefail - source .buildkite/scripts/rocm/stack.sh "$${ROCM_BASE_DOCKERFILE}" - buildkite-agent artifact download "$${ROCM_STACK_BASE_WHEELS_DIR}/*.whl" . - buildkite-agent artifact download "$${ROCM_STACK_VLLM_WHEEL_DIR}/*.whl" . - bash .buildkite/scripts/upload-rocm-wheels.sh "$${ROCM_BASE_DOCKERFILE}" - - - label: ":memo: Annotate ROCm 10.0 wheel release" - id: annotate-rocm100-release - depends_on: upload-rocm100-wheels - agents: *rocm_build_agents - env: *rocm100_env - commands: &rocm_annotate_commands - - bash .buildkite/scripts/annotate-rocm-release.sh "$${ROCM_BASE_DOCKERFILE}" $${ROCM_DEFAULT_ARG} - - # Generate https://wheels.vllm.ai/rocm/ (the default stack's variant), so users can - # `uv pip install vllm --extra-index-url https://wheels.vllm.ai/rocm/` (release only) - - block: "Generate Root Index for ROCm Wheels for Release" - key: block-generate-root-index-rocm-wheels - depends_on: - - upload-rocm100-wheels - - upload-rocm72-wheels - - - label: ":package: Generate Root Index for ROCm Wheels for Release" - depends_on: block-generate-root-index-rocm-wheels - id: generate-root-index-rocm-wheels - agents: *rocm_build_agents - env: *rocm100_env - commands: - - "bash tools/vllm-rocm/generate-rocm-wheels-root-index.sh" - - - label: ":docker: Build release image - x86_64 - ROCm 10.0" - id: build-rocm100-release-image - depends_on: - - step: block-build-release-images - allow_failure: true - - step: build-rocm100-base - allow_failure: false - agents: *rocm_build_agents - timeout_in_minutes: 180 - env: *rocm100_env - commands: &rocm_release_image_commands - - | - set -euo pipefail - source .buildkite/scripts/rocm/stack.sh "$${ROCM_BASE_DOCKERFILE}" - aws ecr-public get-login-password --region us-east-1 | \ - docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7 - docker pull "$${ROCM_STACK_ECR_BASE}" - docker build --file "$${ROCM_STACK_DOCKERFILE}" --target vllm-openai \ - --tag "$${ROCM_STACK_ECR_IMAGE}" --progress plain \ - --build-arg max_jobs=16 --build-arg BASE_IMAGE="$${ROCM_STACK_ECR_BASE}" \ - --build-arg INSTALL_LMCACHE=true --build-arg USE_SCCACHE=1 \ - --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \ - --build-arg SCCACHE_REGION_NAME=us-west-2 --build-arg SCCACHE_S3_NO_CREDENTIALS=0 . - docker push "$${ROCM_STACK_ECR_IMAGE}" - - - label: "Publish nightly ROCm 10.0 image to DockerHub" - id: publish-rocm100-nightly - depends_on: build-rocm100-release-image - if: build.env("NIGHTLY") == "1" - agents: &rocm_publish_agents - queue: small_cpu_queue_release - env: *rocm100_env - commands: &rocm_publish_nightly_commands - - bash .buildkite/scripts/push-nightly-builds-rocm.sh "$${ROCM_BASE_DOCKERFILE}" $${ROCM_DEFAULT_ARG} - plugins: &rocm_dockerhub_login - - docker-login#v3.0.0: - username: vllmbot - password-env: DOCKERHUB_TOKEN - - - group: "Build ROCm 7.2 Wheel / Image" - key: "build-rocm72" - depends_on: ~ - steps: - - label: ":rocm: Build ROCm 7.2 Base Image & Wheels" - id: build-rocm72-base - depends_on: ~ - agents: *rocm_build_agents - timeout_in_minutes: 360 - env: &rocm72_env - ROCM_BASE_DOCKERFILE: "docker/Dockerfile.rocm_72_base" - ROCM_DEFAULT_ARG: "" - DOCKER_BUILDKIT: "1" - S3_BUCKET: "vllm-wheels" - DOCKERHUB_USERNAME: "vllmbot" - commands: *rocm_base_commands - - - label: ":python: Build vLLM ROCm 7.2 Wheel - x86_64" - id: build-rocm72-vllm-wheel - depends_on: build-rocm72-base - agents: *rocm_build_agents - timeout_in_minutes: 180 - env: *rocm72_env - commands: *rocm_vllm_wheel_commands - - - label: ":s3: Upload ROCm 7.2 Wheels to S3" - id: upload-rocm72-wheels - depends_on: build-rocm72-vllm-wheel - agents: *rocm_build_agents - timeout_in_minutes: 60 - env: *rocm72_env - commands: *rocm_upload_commands - - - label: ":memo: Annotate ROCm 7.2 wheel release" - id: annotate-rocm72-release - depends_on: upload-rocm72-wheels - agents: *rocm_build_agents - env: *rocm72_env - commands: *rocm_annotate_commands - - - label: ":docker: Build release image - x86_64 - ROCm 7.2" - id: build-rocm72-release-image - depends_on: - - step: block-build-release-images - allow_failure: true - - step: build-rocm72-base - allow_failure: false - agents: *rocm_build_agents - timeout_in_minutes: 180 - env: *rocm72_env - commands: *rocm_release_image_commands - - - label: "Publish nightly ROCm 7.2 image to DockerHub" - id: publish-rocm72-nightly - depends_on: build-rocm72-release-image - if: build.env("NIGHTLY") == "1" - agents: *rocm_publish_agents - env: *rocm72_env - commands: *rocm_publish_nightly_commands - plugins: *rocm_dockerhub_login - - - label: "Publish nightly XPU image to DockerHub" - depends_on: - - create-manifest-xpu - if: build.env("NIGHTLY") == "1" - agents: - queue: small_cpu_queue_release - commands: - - "bash .buildkite/scripts/xpu/push-nightly-builds-xpu.sh" - - "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly- vllm/vllm-openai-xpu" - plugins: - - docker-login#v3.0.0: - username: vllmbot - password-env: DOCKERHUB_TOKEN - env: - DOCKER_BUILDKIT: "1" - DOCKERHUB_USERNAME: "vllmbot" - - # ============================================================================= - # Publish to DockerHub and PyPI (at the end so all builds complete first) - # ============================================================================= - - - block: "Publish release images to DockerHub" - key: block-publish-release-images - depends_on: - - create-multi-arch-manifest - - create-multi-arch-manifest-cuda-12-9 - - create-multi-arch-manifest-ubuntu2404 - - create-multi-arch-manifest-cuda-12-9-ubuntu2404 - - create-manifest-xpu - - build-rocm100-release-image - - build-rocm72-release-image - - input-release-version - # Wait for CPU builds if their block steps were unblocked, so publish - # doesn't race the in-progress CPU build. allow_failure lets publish - # proceed when the operator legitimately leaves the CPU block steps - # unblocked or the CPU build fails. - - step: build-cpu-release-image-x86 - allow_failure: true - - step: build-cpu-release-image-arm64 - allow_failure: true - - - label: "Publish {{matrix}} release images to DockerHub" - depends_on: - - block-publish-release-images - key: publish-release-images-dockerhub - agents: - queue: small_cpu_queue_release - commands: - - "bash .buildkite/scripts/publish-release-images.sh {{matrix}}" - matrix: - - "cuda-13-0" - - "cuda-12-9" - - "cuda-13-0-ubuntu-24-04" - - "cuda-12-9-ubuntu-24-04" - - "rocm" - - "rocm72" - - "xpu" - - "cpu" - plugins: - - docker-login#v3.0.0: - username: vllmbot - password-env: DOCKERHUB_TOKEN - env: - DOCKER_BUILDKIT: "1" - DOCKERHUB_USERNAME: "vllmbot" - - - group: "Publish wheels" - key: "publish-wheels" - steps: - - block: "Confirm update release wheels to PyPI (experimental, use with caution)?" - key: block-upload-release-wheels - depends_on: - - input-release-version - - build-wheels - - build-additional-wheels - - generate-additional-wheel-indices - - - label: "Upload release wheels to PyPI" - depends_on: - - block-upload-release-wheels - id: upload-release-wheels - agents: - queue: small_cpu_queue_release - commands: - - "bash .buildkite/scripts/upload-release-wheels-pypi.sh" diff --git a/.buildkite/scripts/annotate-build-artifact.sh b/.buildkite/scripts/annotate-build-artifact.sh deleted file mode 100755 index 3a8046ed2e..0000000000 --- a/.buildkite/scripts/annotate-build-artifact.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# -# Append a build artifact line to the Buildkite annotation. -# Usage: annotate-build-artifact.sh