[CI] Mooncake PD integration tests (#46844)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-08-03 06:55:51 -07:00
committed by GitHub
parent 005fa01756
commit 8ba87e0181
9 changed files with 413 additions and 4 deletions
+19 -3
View File
@@ -13,7 +13,7 @@ REQUIREMENTS_FILE="${KV_CONNECTORS_REQUIREMENTS:-/vllm-workspace/requirements/kv
uv pip install --system -r "${REQUIREMENTS_FILE}"
NIXL_METADATA=$(python3 - <<'PY'
KV_METADATA=$(python3 - <<'PY'
import importlib.metadata as metadata
import torch
@@ -22,10 +22,16 @@ cuda_version = torch.version.cuda
if cuda_version is None:
raise SystemExit("torch.version.cuda is not set")
print(cuda_version.split(".", 1)[0], metadata.version("nixl"))
try:
mooncake_version = metadata.version("mooncake-transfer-engine")
except metadata.PackageNotFoundError:
mooncake_version = ""
print(cuda_version.split(".", 1)[0], metadata.version("nixl"), mooncake_version)
PY
)
read -r CUDA_MAJOR NIXL_VERSION <<<"${NIXL_METADATA}"
read -r CUDA_MAJOR NIXL_VERSION MOONCAKE_VERSION <<<"${KV_METADATA}"
MOONCAKE_VERSION="${MOONCAKE_VERSION:-}"
# nixl>=1.1.0 can install multiple CUDA wheel variants. Keep only the variant
# matching this CI image so nixl_ep_cpp links against the available libcudart.
@@ -42,3 +48,13 @@ for package_name in ("nixl", "nixl-cu12", "nixl-cu13"):
version = "not installed"
print(f"{package_name}: {version}")
PY
# The default mooncake-transfer-engine PyPI wheel is built against CUDA 12; its
# engine.so links libcudart.so.12, absent from the CUDA 13 runtime image. On a
# CUDA 13 image, swap it for the cuda13 variant (same version), which links
# libcudart.so.13. Both expose the `mooncake` package, so uninstall the CUDA 12
# build first to avoid a clash.
if [ "${CUDA_MAJOR}" = "13" ] && [ -n "${MOONCAKE_VERSION}" ]; then
uv pip uninstall --system mooncake-transfer-engine 2>/dev/null || true
uv pip install --system "mooncake-transfer-engine-cuda13==${MOONCAKE_VERSION}"
fi
@@ -0,0 +1,21 @@
group: Disaggregated Mooncake
depends_on:
- image-build
steps:
- label: Distributed MooncakeConnector PD accuracy (4 GPUs)
key: distributed-mooncakeconnector-pd-accuracy-4-gpus
timeout_in_minutes: 10
working_dir: "/vllm-workspace/tests"
# Runs on b200-k8s because it is the only NVIDIA pool that can carry mooncake
# transfers: its pods are the only ones granted IPC_LOCK, which RDMA needs to
# pin memory, and the only ones using host networking. Elsewhere topology
# discovery finds no usable HCA and the tcp fallback then advertises the
# docker bridge address, so every transfer fails to connect.
device: b200-k8s
num_devices: 4
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/mooncake/
- tests/v1/kv_connector/mooncake_integration/
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/mooncake_integration/config_sweep_accuracy_test.sh
@@ -60,6 +60,7 @@ Now you can send requests to the proxy server through port 8000.
- **num_workers**: Size of thread pool for one prefiller worker to transfer KV caches by mooncake. (default 10)
- **mooncake_protocol**: Mooncake connector protocol. (default "rdma")
- **device_name**: Comma-separated whitelist of RDMA devices (e.g. `"mlx5_0,mlx5_1"`) to restrict topology discovery to. Empty discovers every device. Useful on hosts exposing a mix of InfiniBand and RoCE ports, where both peers must settle on the same link layer.
## Example Scripts/Code
+3 -1
View File
@@ -3,4 +3,6 @@ lmcache >= 0.3.9
# until a fixed newer release is verified for runtime images.
cupy-cuda13x < 14.1.0
nixl == 1.3.1
mooncake-transfer-engine >= 0.3.8
# CUDA 12 build. On the CUDA 13 image install-kv-connectors.sh swaps this for
# the mooncake-transfer-engine-cuda13 variant of the same version.
mooncake-transfer-engine >= 0.3.12
@@ -0,0 +1,53 @@
#!/usr/bin/env bash
set -euo pipefail
# Utility to run Mooncake P/D integration tests sequentially with varying TP
# configurations.
SCRIPT="v1/kv_connector/mooncake_integration/run_accuracy_test.sh"
IMPORT_CANARY="v1/kv_connector/mooncake_integration/test_mooncake_imports.py"
echo "=== Running Mooncake import canary ==="
python3 -m pytest -s -x "${IMPORT_CANARY}"
# Symmetric TP configurations (default model: Qwen/Qwen3-0.6B).
tp_configs=(
"GPU_MEMORY_UTILIZATION=0.6 PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=1"
"GPU_MEMORY_UTILIZATION=0.6 PREFILLER_TP_SIZE=2 DECODER_TP_SIZE=2"
)
configs=("${tp_configs[@]}")
run_tests() {
local label=$1
local extra_args=$2
echo "=== Running tests (${label}) ==="
for cfg in "${configs[@]}"; do
local -a cfg_parts extra_args_parts
read -r -a cfg_parts <<< "$cfg"
read -r -a extra_args_parts <<< "$extra_args"
echo "-> Running with ${cfg} ${extra_args:+and ${extra_args}}"
# Use 'env' to safely set variables without eval
# keep argv splitting safe and SC2086-clean via arrays.
if ! env "${cfg_parts[@]}" bash "${SCRIPT}" "${extra_args_parts[@]}"; then
echo "❌ Test failed for config: ${cfg} ${extra_args:+(${extra_args})}"
exit 1
fi
done
echo "✅ All ${label} tests passed!"
}
# Set backend
label="default backend"
cmdline_args=""
if [[ -n "${ATTENTION_BACKEND:-}" ]]; then
echo "ATTENTION_BACKEND is set, running with --attention-backend ${ATTENTION_BACKEND}"
label="${ATTENTION_BACKEND} backend"
cmdline_args=" --attention-backend ${ATTENTION_BACKEND} "
else
echo "running with default attention backend"
fi
# Run tests
run_tests "${label}" "${cmdline_args}"
@@ -0,0 +1,227 @@
#!/bin/bash
set -xe
# Parse command line arguments
ATTENTION_BACKEND="" # Default to empty (use vllm default)
while [[ $# -gt 0 ]]; do
case $1 in
--attention-backend)
ATTENTION_BACKEND="$2"
shift 2
;;
*)
echo "Unknown option $1"
echo "Usage: $0 [--attention-backend <backend>]"
exit 1
;;
esac
done
if [[ -n "$ATTENTION_BACKEND" ]]; then
echo "Using attention backend: $ATTENTION_BACKEND"
fi
if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then
echo "vLLM serve extra args: $VLLM_SERVE_EXTRA_ARGS"
fi
# Mooncake discovers peers via its bootstrap server, so no side-channel
# or layout env vars are needed.
# Comma-separated whitelist restricting RDMA topology discovery to named HCAs.
# This default is specific to the b200-k8s CI pool: those nodes expose four
# InfiniBand HCAs that have no network device next to a single RoCE port
# (mlx5_12). With all five visible the two sides can settle on different link
# layers, and the QP handshake then fails to reach RTR. Set to a different NIC
# on other hardware, or to the empty string to discover every device.
MOONCAKE_DEVICE_NAME=${MOONCAKE_DEVICE_NAME-mlx5_12}
kv_config() {
printf '{"kv_connector":"MooncakeConnector","kv_role":"%s",' "$1"
printf '"kv_connector_extra_config":{"device_name":"%s"}}' "${MOONCAKE_DEVICE_NAME}"
}
KV_CONFIG_P=$(kv_config kv_producer)
KV_CONFIG_D=$(kv_config kv_consumer)
# Models to run
MODEL_NAME=${MODEL_NAME:-}
if [[ -n "$MODEL_NAME" ]]; then
MODEL="$MODEL_NAME"
else
MODEL="Qwen/Qwen3-0.6B"
fi
# Number of prefill and decode instances to create
NUM_PREFILL_INSTANCES=${NUM_PREFILL_INSTANCES:-1} # Default to 1
NUM_DECODE_INSTANCES=${NUM_DECODE_INSTANCES:-1} # Default to 1
PREFILLER_TP_SIZE=${PREFILLER_TP_SIZE:-1}
DECODER_TP_SIZE=${DECODER_TP_SIZE:-1}
GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.2}
BLOCK_SIZE=${BLOCK_SIZE:-128}
# Comma-separated extra args for vllm serve (e.g. --max-model-len,2048)
VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-}
# Resolve the repository root from the script location instead of `.git`.
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}"
SMI_BIN=$(which nvidia-smi || which rocm-smi || echo "")
# Trap the SIGINT signal (triggered by Ctrl+C)
trap 'kill $(jobs -pr)' SIGINT SIGTERM EXIT
# Waits for vLLM to start.
wait_for_server() {
local port=$1
timeout 1200 bash -c "
until curl -s localhost:${port}/v1/completions > /dev/null; do
sleep 1
done" && return 0 || return 1
}
# Function to clean up previous instances
cleanup_instances() {
echo "Cleaning up any running vLLM instances..."
pkill -f "vllm serve" || true
sleep 2
}
get_num_gpus() {
if [[ "$SMI_BIN" == *"nvidia"* ]]; then
$SMI_BIN --query-gpu=name --format=csv,noheader | wc -l
elif [[ "$SMI_BIN" == *"rocm"* ]]; then
$SMI_BIN -l | grep -c GPU
else
# works for non-cuda platforms,
# assuming at least 1 device and
# let system to decide which card to use
echo "1"
fi
}
# Function to run tests for a specific model
run_tests_for_model() {
local model_name=$1
echo "================================"
echo "Testing model: $model_name"
echo "================================"
# Proxy args: repeated --prefill URL BOOTSTRAP_PORT and --decode URL.
PROXY_ARGS=()
# Start prefill instances
for i in $(seq 0 $((NUM_PREFILL_INSTANCES-1))); do
# Calculate GPU ID - we'll distribute across available GPUs
GPU_ID=$((i % $(get_num_gpus)))
NEXT_GPU=${GPU_ID}
# If PREFILLER_TP_SIZE is more than 1
for (( j=1; j < PREFILLER_TP_SIZE; j++ )); do
NEXT_GPU=$(((GPU_ID + j) % $(get_num_gpus)))
GPU_ID="${GPU_ID},${NEXT_GPU}"
done
# Calculate port number (base port + instance number)
PORT=$((8100 + i))
# Bootstrap server port for this prefiller. Avoid clash across instances.
BOOTSTRAP_PORT=$((8998 + i))
echo "Starting prefill instance $i on GPU $GPU_ID, port $PORT, bootstrap $BOOTSTRAP_PORT"
BASE_CMD="CUDA_VISIBLE_DEVICES=$GPU_ID \
VLLM_MOONCAKE_BOOTSTRAP_PORT=$BOOTSTRAP_PORT \
vllm serve $model_name \
--port $PORT \
--enforce-eager \
--block-size ${BLOCK_SIZE} \
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
--tensor-parallel-size $PREFILLER_TP_SIZE \
--kv-transfer-config '$KV_CONFIG_P'"
if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then
IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS"
for arg in "${extra_args[@]}"; do
BASE_CMD="${BASE_CMD} $arg"
done
fi
# Add attention backend config if specified
if [[ -n "$ATTENTION_BACKEND" ]]; then
BASE_CMD="${BASE_CMD} --attention-backend=$ATTENTION_BACKEND"
fi
eval "$BASE_CMD &"
PROXY_ARGS+=(--prefill "http://localhost:${PORT}" "$BOOTSTRAP_PORT")
PREFILL_PORTS+=("$PORT")
done
# Start decode instances
for i in $(seq 0 $((NUM_DECODE_INSTANCES-1))); do
# Calculate GPU ID - distribute starting from after prefill GPUs
GPU_ID=$(((i + NEXT_GPU + 1) % $(get_num_gpus)))
# If DECODER_TP_SIZE is more than 1
for (( j=1; j < DECODER_TP_SIZE; j++ )); do
NEXT_GPU=$(((GPU_ID + j) % $(get_num_gpus)))
GPU_ID="${GPU_ID},${NEXT_GPU}"
done
# Calculate port number (base port + instance number)
PORT=$((8200 + i))
echo "Starting decode instance $i on GPU $GPU_ID, port $PORT"
BASE_CMD="CUDA_VISIBLE_DEVICES=$GPU_ID \
vllm serve $model_name \
--port $PORT \
--enforce-eager \
--block-size ${BLOCK_SIZE} \
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
--tensor-parallel-size $DECODER_TP_SIZE \
--kv-transfer-config '$KV_CONFIG_D'"
if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then
IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS"
for arg in "${extra_args[@]}"; do
BASE_CMD="${BASE_CMD} $arg"
done
fi
# Add attention backend config if specified
if [[ -n "$ATTENTION_BACKEND" ]]; then
BASE_CMD="${BASE_CMD} --attention-backend=$ATTENTION_BACKEND"
fi
eval "$BASE_CMD &"
PROXY_ARGS+=(--decode "http://localhost:${PORT}")
DECODE_PORTS+=("$PORT")
done
# Wait for all instances to start
for PORT in "${PREFILL_PORTS[@]}"; do
echo "Waiting for prefill instance on port $PORT to start..."
wait_for_server "$PORT"
done
for PORT in "${DECODE_PORTS[@]}"; do
echo "Waiting for decode instance on port $PORT to start..."
wait_for_server "$PORT"
done
# Start the proxy server
echo "Starting proxy server on port 8192"
python3 "${GIT_ROOT}/examples/disaggregated/mooncake_connector/mooncake_connector_proxy.py" \
--port 8192 "${PROXY_ARGS[@]}" &
# Wait for the proxy to start
sleep 5
# Run lm eval for this model
echo "Running tests for $model_name"
TEST_MODEL=$model_name python3 -m pytest -s -x \
"${GIT_ROOT}/tests/v1/kv_connector/mooncake_integration/test_accuracy.py"
# Clean up before running next model
cleanup_instances
sleep 3
}
run_tests_for_model "$MODEL"
echo "All tests completed!"
@@ -0,0 +1,67 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import lm_eval
import openai
BASE_URL = "http://localhost:8192/v1"
NUM_CONCURRENT = 100
TASK = "gsm8k"
FILTER = "exact_match,strict-match"
RTOL = 0.05
# Model-specific expected values
EXPECTED_VALUES = {
"Qwen/Qwen3-0.6B": 0.41,
}
SIMPLE_PROMPT = (
"The best part about working on vLLM is that I got to meet so many people across "
"various different organizations like UCB, Google, and Meta which means",
)
# Get model name from environment variable
MODEL_NAME = os.environ.get("TEST_MODEL", "Qwen/Qwen3-0.6B")
def run_simple_prompt():
client = openai.OpenAI(api_key="EMPTY", base_url=BASE_URL)
completion = client.completions.create(model=MODEL_NAME, prompt=SIMPLE_PROMPT)
print("-" * 50)
print(f"Completion results for {MODEL_NAME}:")
print(completion)
print("-" * 50)
def test_accuracy():
"""Run the end to end accuracy test."""
run_simple_prompt()
model_args = (
f"model={MODEL_NAME},"
f"base_url={BASE_URL}/completions,"
f"num_concurrent={NUM_CONCURRENT},tokenized_requests=False"
)
results = lm_eval.simple_evaluate(
model="local-completions",
model_args=model_args,
tasks=TASK,
)
measured_value = results["results"][TASK][FILTER]
expected_value = EXPECTED_VALUES.get(MODEL_NAME)
print(f"Measured accuracy value: {measured_value}\n")
if expected_value is None:
print(
f"Warning: No expected value found for {MODEL_NAME}. "
"Skipping accuracy check."
)
return
assert (
measured_value - RTOL < expected_value
and measured_value + RTOL > expected_value
), f"Expected: {expected_value} | Measured: {measured_value}"
@@ -0,0 +1,22 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Mooncake import canary for CUDA wheel selection."""
import importlib.metadata as metadata
def test_mooncake_engine_imports():
"""Fail here rather than as "Mooncake is not available" at engine startup.
The PyPI mooncake-transfer-engine wheel is built against CUDA 12, so on the
CUDA 13 image install-kv-connectors.sh replaces it with the cuda13 variant.
A mismatch surfaces as an ImportError on the compiled extension.
"""
for package_name in ("mooncake-transfer-engine", "mooncake-transfer-engine-cuda13"):
try:
version = metadata.version(package_name)
except metadata.PackageNotFoundError:
version = "not installed"
print(f"{package_name}: {version}")
import mooncake.engine # noqa: F401