[CI] Mooncake PD integration tests (#46844)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -13,7 +13,7 @@ REQUIREMENTS_FILE="${KV_CONNECTORS_REQUIREMENTS:-/vllm-workspace/requirements/kv
|
||||
|
||||
uv pip install --system -r "${REQUIREMENTS_FILE}"
|
||||
|
||||
NIXL_METADATA=$(python3 - <<'PY'
|
||||
KV_METADATA=$(python3 - <<'PY'
|
||||
import importlib.metadata as metadata
|
||||
|
||||
import torch
|
||||
@@ -22,10 +22,16 @@ cuda_version = torch.version.cuda
|
||||
if cuda_version is None:
|
||||
raise SystemExit("torch.version.cuda is not set")
|
||||
|
||||
print(cuda_version.split(".", 1)[0], metadata.version("nixl"))
|
||||
try:
|
||||
mooncake_version = metadata.version("mooncake-transfer-engine")
|
||||
except metadata.PackageNotFoundError:
|
||||
mooncake_version = ""
|
||||
|
||||
print(cuda_version.split(".", 1)[0], metadata.version("nixl"), mooncake_version)
|
||||
PY
|
||||
)
|
||||
read -r CUDA_MAJOR NIXL_VERSION <<<"${NIXL_METADATA}"
|
||||
read -r CUDA_MAJOR NIXL_VERSION MOONCAKE_VERSION <<<"${KV_METADATA}"
|
||||
MOONCAKE_VERSION="${MOONCAKE_VERSION:-}"
|
||||
|
||||
# nixl>=1.1.0 can install multiple CUDA wheel variants. Keep only the variant
|
||||
# matching this CI image so nixl_ep_cpp links against the available libcudart.
|
||||
@@ -42,3 +48,13 @@ for package_name in ("nixl", "nixl-cu12", "nixl-cu13"):
|
||||
version = "not installed"
|
||||
print(f"{package_name}: {version}")
|
||||
PY
|
||||
|
||||
# The default mooncake-transfer-engine PyPI wheel is built against CUDA 12; its
|
||||
# engine.so links libcudart.so.12, absent from the CUDA 13 runtime image. On a
|
||||
# CUDA 13 image, swap it for the cuda13 variant (same version), which links
|
||||
# libcudart.so.13. Both expose the `mooncake` package, so uninstall the CUDA 12
|
||||
# build first to avoid a clash.
|
||||
if [ "${CUDA_MAJOR}" = "13" ] && [ -n "${MOONCAKE_VERSION}" ]; then
|
||||
uv pip uninstall --system mooncake-transfer-engine 2>/dev/null || true
|
||||
uv pip install --system "mooncake-transfer-engine-cuda13==${MOONCAKE_VERSION}"
|
||||
fi
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
group: Disaggregated Mooncake
|
||||
depends_on:
|
||||
- image-build
|
||||
steps:
|
||||
- label: Distributed MooncakeConnector PD accuracy (4 GPUs)
|
||||
key: distributed-mooncakeconnector-pd-accuracy-4-gpus
|
||||
timeout_in_minutes: 10
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
# Runs on b200-k8s because it is the only NVIDIA pool that can carry mooncake
|
||||
# transfers: its pods are the only ones granted IPC_LOCK, which RDMA needs to
|
||||
# pin memory, and the only ones using host networking. Elsewhere topology
|
||||
# discovery finds no usable HCA and the tcp fallback then advertises the
|
||||
# docker bridge address, so every transfer fails to connect.
|
||||
device: b200-k8s
|
||||
num_devices: 4
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/mooncake/
|
||||
- tests/v1/kv_connector/mooncake_integration/
|
||||
commands:
|
||||
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
|
||||
- bash v1/kv_connector/mooncake_integration/config_sweep_accuracy_test.sh
|
||||
@@ -60,6 +60,7 @@ Now you can send requests to the proxy server through port 8000.
|
||||
|
||||
- **num_workers**: Size of thread pool for one prefiller worker to transfer KV caches by mooncake. (default 10)
|
||||
- **mooncake_protocol**: Mooncake connector protocol. (default "rdma")
|
||||
- **device_name**: Comma-separated whitelist of RDMA devices (e.g. `"mlx5_0,mlx5_1"`) to restrict topology discovery to. Empty discovers every device. Useful on hosts exposing a mix of InfiniBand and RoCE ports, where both peers must settle on the same link layer.
|
||||
|
||||
## Example Scripts/Code
|
||||
|
||||
|
||||
@@ -3,4 +3,6 @@ lmcache >= 0.3.9
|
||||
# until a fixed newer release is verified for runtime images.
|
||||
cupy-cuda13x < 14.1.0
|
||||
nixl == 1.3.1
|
||||
mooncake-transfer-engine >= 0.3.8
|
||||
# CUDA 12 build. On the CUDA 13 image install-kv-connectors.sh swaps this for
|
||||
# the mooncake-transfer-engine-cuda13 variant of the same version.
|
||||
mooncake-transfer-engine >= 0.3.12
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Utility to run Mooncake P/D integration tests sequentially with varying TP
|
||||
# configurations.
|
||||
SCRIPT="v1/kv_connector/mooncake_integration/run_accuracy_test.sh"
|
||||
IMPORT_CANARY="v1/kv_connector/mooncake_integration/test_mooncake_imports.py"
|
||||
|
||||
echo "=== Running Mooncake import canary ==="
|
||||
python3 -m pytest -s -x "${IMPORT_CANARY}"
|
||||
|
||||
# Symmetric TP configurations (default model: Qwen/Qwen3-0.6B).
|
||||
tp_configs=(
|
||||
"GPU_MEMORY_UTILIZATION=0.6 PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=1"
|
||||
"GPU_MEMORY_UTILIZATION=0.6 PREFILLER_TP_SIZE=2 DECODER_TP_SIZE=2"
|
||||
)
|
||||
|
||||
configs=("${tp_configs[@]}")
|
||||
|
||||
run_tests() {
|
||||
local label=$1
|
||||
local extra_args=$2
|
||||
|
||||
echo "=== Running tests (${label}) ==="
|
||||
for cfg in "${configs[@]}"; do
|
||||
local -a cfg_parts extra_args_parts
|
||||
read -r -a cfg_parts <<< "$cfg"
|
||||
read -r -a extra_args_parts <<< "$extra_args"
|
||||
|
||||
echo "-> Running with ${cfg} ${extra_args:+and ${extra_args}}"
|
||||
# Use 'env' to safely set variables without eval
|
||||
# keep argv splitting safe and SC2086-clean via arrays.
|
||||
if ! env "${cfg_parts[@]}" bash "${SCRIPT}" "${extra_args_parts[@]}"; then
|
||||
echo "❌ Test failed for config: ${cfg} ${extra_args:+(${extra_args})}"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "✅ All ${label} tests passed!"
|
||||
}
|
||||
|
||||
# Set backend
|
||||
label="default backend"
|
||||
cmdline_args=""
|
||||
if [[ -n "${ATTENTION_BACKEND:-}" ]]; then
|
||||
echo "ATTENTION_BACKEND is set, running with --attention-backend ${ATTENTION_BACKEND}"
|
||||
label="${ATTENTION_BACKEND} backend"
|
||||
cmdline_args=" --attention-backend ${ATTENTION_BACKEND} "
|
||||
else
|
||||
echo "running with default attention backend"
|
||||
fi
|
||||
|
||||
# Run tests
|
||||
run_tests "${label}" "${cmdline_args}"
|
||||
@@ -0,0 +1,227 @@
|
||||
#!/bin/bash
|
||||
set -xe
|
||||
|
||||
# Parse command line arguments
|
||||
ATTENTION_BACKEND="" # Default to empty (use vllm default)
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--attention-backend)
|
||||
ATTENTION_BACKEND="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option $1"
|
||||
echo "Usage: $0 [--attention-backend <backend>]"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -n "$ATTENTION_BACKEND" ]]; then
|
||||
echo "Using attention backend: $ATTENTION_BACKEND"
|
||||
fi
|
||||
if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then
|
||||
echo "vLLM serve extra args: $VLLM_SERVE_EXTRA_ARGS"
|
||||
fi
|
||||
|
||||
# Mooncake discovers peers via its bootstrap server, so no side-channel
|
||||
# or layout env vars are needed.
|
||||
|
||||
# Comma-separated whitelist restricting RDMA topology discovery to named HCAs.
|
||||
# This default is specific to the b200-k8s CI pool: those nodes expose four
|
||||
# InfiniBand HCAs that have no network device next to a single RoCE port
|
||||
# (mlx5_12). With all five visible the two sides can settle on different link
|
||||
# layers, and the QP handshake then fails to reach RTR. Set to a different NIC
|
||||
# on other hardware, or to the empty string to discover every device.
|
||||
MOONCAKE_DEVICE_NAME=${MOONCAKE_DEVICE_NAME-mlx5_12}
|
||||
kv_config() {
|
||||
printf '{"kv_connector":"MooncakeConnector","kv_role":"%s",' "$1"
|
||||
printf '"kv_connector_extra_config":{"device_name":"%s"}}' "${MOONCAKE_DEVICE_NAME}"
|
||||
}
|
||||
KV_CONFIG_P=$(kv_config kv_producer)
|
||||
KV_CONFIG_D=$(kv_config kv_consumer)
|
||||
|
||||
# Models to run
|
||||
MODEL_NAME=${MODEL_NAME:-}
|
||||
if [[ -n "$MODEL_NAME" ]]; then
|
||||
MODEL="$MODEL_NAME"
|
||||
else
|
||||
MODEL="Qwen/Qwen3-0.6B"
|
||||
fi
|
||||
|
||||
# Number of prefill and decode instances to create
|
||||
NUM_PREFILL_INSTANCES=${NUM_PREFILL_INSTANCES:-1} # Default to 1
|
||||
NUM_DECODE_INSTANCES=${NUM_DECODE_INSTANCES:-1} # Default to 1
|
||||
PREFILLER_TP_SIZE=${PREFILLER_TP_SIZE:-1}
|
||||
DECODER_TP_SIZE=${DECODER_TP_SIZE:-1}
|
||||
GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.2}
|
||||
BLOCK_SIZE=${BLOCK_SIZE:-128}
|
||||
# Comma-separated extra args for vllm serve (e.g. --max-model-len,2048)
|
||||
VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-}
|
||||
|
||||
# Resolve the repository root from the script location instead of `.git`.
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
|
||||
GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}"
|
||||
|
||||
SMI_BIN=$(which nvidia-smi || which rocm-smi || echo "")
|
||||
|
||||
# Trap the SIGINT signal (triggered by Ctrl+C)
|
||||
trap 'kill $(jobs -pr)' SIGINT SIGTERM EXIT
|
||||
|
||||
# Waits for vLLM to start.
|
||||
wait_for_server() {
|
||||
local port=$1
|
||||
timeout 1200 bash -c "
|
||||
until curl -s localhost:${port}/v1/completions > /dev/null; do
|
||||
sleep 1
|
||||
done" && return 0 || return 1
|
||||
}
|
||||
|
||||
# Function to clean up previous instances
|
||||
cleanup_instances() {
|
||||
echo "Cleaning up any running vLLM instances..."
|
||||
pkill -f "vllm serve" || true
|
||||
sleep 2
|
||||
}
|
||||
|
||||
get_num_gpus() {
|
||||
if [[ "$SMI_BIN" == *"nvidia"* ]]; then
|
||||
$SMI_BIN --query-gpu=name --format=csv,noheader | wc -l
|
||||
elif [[ "$SMI_BIN" == *"rocm"* ]]; then
|
||||
$SMI_BIN -l | grep -c GPU
|
||||
else
|
||||
# works for non-cuda platforms,
|
||||
# assuming at least 1 device and
|
||||
# let system to decide which card to use
|
||||
echo "1"
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to run tests for a specific model
|
||||
run_tests_for_model() {
|
||||
local model_name=$1
|
||||
echo "================================"
|
||||
echo "Testing model: $model_name"
|
||||
echo "================================"
|
||||
|
||||
# Proxy args: repeated --prefill URL BOOTSTRAP_PORT and --decode URL.
|
||||
PROXY_ARGS=()
|
||||
|
||||
# Start prefill instances
|
||||
for i in $(seq 0 $((NUM_PREFILL_INSTANCES-1))); do
|
||||
# Calculate GPU ID - we'll distribute across available GPUs
|
||||
GPU_ID=$((i % $(get_num_gpus)))
|
||||
NEXT_GPU=${GPU_ID}
|
||||
# If PREFILLER_TP_SIZE is more than 1
|
||||
for (( j=1; j < PREFILLER_TP_SIZE; j++ )); do
|
||||
NEXT_GPU=$(((GPU_ID + j) % $(get_num_gpus)))
|
||||
GPU_ID="${GPU_ID},${NEXT_GPU}"
|
||||
done
|
||||
|
||||
# Calculate port number (base port + instance number)
|
||||
PORT=$((8100 + i))
|
||||
# Bootstrap server port for this prefiller. Avoid clash across instances.
|
||||
BOOTSTRAP_PORT=$((8998 + i))
|
||||
|
||||
echo "Starting prefill instance $i on GPU $GPU_ID, port $PORT, bootstrap $BOOTSTRAP_PORT"
|
||||
|
||||
BASE_CMD="CUDA_VISIBLE_DEVICES=$GPU_ID \
|
||||
VLLM_MOONCAKE_BOOTSTRAP_PORT=$BOOTSTRAP_PORT \
|
||||
vllm serve $model_name \
|
||||
--port $PORT \
|
||||
--enforce-eager \
|
||||
--block-size ${BLOCK_SIZE} \
|
||||
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
|
||||
--tensor-parallel-size $PREFILLER_TP_SIZE \
|
||||
--kv-transfer-config '$KV_CONFIG_P'"
|
||||
if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then
|
||||
IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS"
|
||||
for arg in "${extra_args[@]}"; do
|
||||
BASE_CMD="${BASE_CMD} $arg"
|
||||
done
|
||||
fi
|
||||
|
||||
# Add attention backend config if specified
|
||||
if [[ -n "$ATTENTION_BACKEND" ]]; then
|
||||
BASE_CMD="${BASE_CMD} --attention-backend=$ATTENTION_BACKEND"
|
||||
fi
|
||||
|
||||
eval "$BASE_CMD &"
|
||||
|
||||
PROXY_ARGS+=(--prefill "http://localhost:${PORT}" "$BOOTSTRAP_PORT")
|
||||
PREFILL_PORTS+=("$PORT")
|
||||
done
|
||||
|
||||
# Start decode instances
|
||||
for i in $(seq 0 $((NUM_DECODE_INSTANCES-1))); do
|
||||
# Calculate GPU ID - distribute starting from after prefill GPUs
|
||||
GPU_ID=$(((i + NEXT_GPU + 1) % $(get_num_gpus)))
|
||||
# If DECODER_TP_SIZE is more than 1
|
||||
for (( j=1; j < DECODER_TP_SIZE; j++ )); do
|
||||
NEXT_GPU=$(((GPU_ID + j) % $(get_num_gpus)))
|
||||
GPU_ID="${GPU_ID},${NEXT_GPU}"
|
||||
done
|
||||
# Calculate port number (base port + instance number)
|
||||
PORT=$((8200 + i))
|
||||
|
||||
echo "Starting decode instance $i on GPU $GPU_ID, port $PORT"
|
||||
|
||||
BASE_CMD="CUDA_VISIBLE_DEVICES=$GPU_ID \
|
||||
vllm serve $model_name \
|
||||
--port $PORT \
|
||||
--enforce-eager \
|
||||
--block-size ${BLOCK_SIZE} \
|
||||
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
|
||||
--tensor-parallel-size $DECODER_TP_SIZE \
|
||||
--kv-transfer-config '$KV_CONFIG_D'"
|
||||
if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then
|
||||
IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS"
|
||||
for arg in "${extra_args[@]}"; do
|
||||
BASE_CMD="${BASE_CMD} $arg"
|
||||
done
|
||||
fi
|
||||
|
||||
# Add attention backend config if specified
|
||||
if [[ -n "$ATTENTION_BACKEND" ]]; then
|
||||
BASE_CMD="${BASE_CMD} --attention-backend=$ATTENTION_BACKEND"
|
||||
fi
|
||||
|
||||
eval "$BASE_CMD &"
|
||||
|
||||
PROXY_ARGS+=(--decode "http://localhost:${PORT}")
|
||||
DECODE_PORTS+=("$PORT")
|
||||
done
|
||||
|
||||
# Wait for all instances to start
|
||||
for PORT in "${PREFILL_PORTS[@]}"; do
|
||||
echo "Waiting for prefill instance on port $PORT to start..."
|
||||
wait_for_server "$PORT"
|
||||
done
|
||||
|
||||
for PORT in "${DECODE_PORTS[@]}"; do
|
||||
echo "Waiting for decode instance on port $PORT to start..."
|
||||
wait_for_server "$PORT"
|
||||
done
|
||||
|
||||
# Start the proxy server
|
||||
echo "Starting proxy server on port 8192"
|
||||
python3 "${GIT_ROOT}/examples/disaggregated/mooncake_connector/mooncake_connector_proxy.py" \
|
||||
--port 8192 "${PROXY_ARGS[@]}" &
|
||||
|
||||
# Wait for the proxy to start
|
||||
sleep 5
|
||||
|
||||
# Run lm eval for this model
|
||||
echo "Running tests for $model_name"
|
||||
TEST_MODEL=$model_name python3 -m pytest -s -x \
|
||||
"${GIT_ROOT}/tests/v1/kv_connector/mooncake_integration/test_accuracy.py"
|
||||
|
||||
# Clean up before running next model
|
||||
cleanup_instances
|
||||
sleep 3
|
||||
}
|
||||
|
||||
run_tests_for_model "$MODEL"
|
||||
|
||||
echo "All tests completed!"
|
||||
@@ -0,0 +1,67 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import os
|
||||
|
||||
import lm_eval
|
||||
import openai
|
||||
|
||||
BASE_URL = "http://localhost:8192/v1"
|
||||
NUM_CONCURRENT = 100
|
||||
TASK = "gsm8k"
|
||||
FILTER = "exact_match,strict-match"
|
||||
RTOL = 0.05
|
||||
|
||||
# Model-specific expected values
|
||||
EXPECTED_VALUES = {
|
||||
"Qwen/Qwen3-0.6B": 0.41,
|
||||
}
|
||||
|
||||
SIMPLE_PROMPT = (
|
||||
"The best part about working on vLLM is that I got to meet so many people across "
|
||||
"various different organizations like UCB, Google, and Meta which means",
|
||||
)
|
||||
|
||||
# Get model name from environment variable
|
||||
MODEL_NAME = os.environ.get("TEST_MODEL", "Qwen/Qwen3-0.6B")
|
||||
|
||||
|
||||
def run_simple_prompt():
|
||||
client = openai.OpenAI(api_key="EMPTY", base_url=BASE_URL)
|
||||
completion = client.completions.create(model=MODEL_NAME, prompt=SIMPLE_PROMPT)
|
||||
|
||||
print("-" * 50)
|
||||
print(f"Completion results for {MODEL_NAME}:")
|
||||
print(completion)
|
||||
print("-" * 50)
|
||||
|
||||
|
||||
def test_accuracy():
|
||||
"""Run the end to end accuracy test."""
|
||||
run_simple_prompt()
|
||||
|
||||
model_args = (
|
||||
f"model={MODEL_NAME},"
|
||||
f"base_url={BASE_URL}/completions,"
|
||||
f"num_concurrent={NUM_CONCURRENT},tokenized_requests=False"
|
||||
)
|
||||
results = lm_eval.simple_evaluate(
|
||||
model="local-completions",
|
||||
model_args=model_args,
|
||||
tasks=TASK,
|
||||
)
|
||||
|
||||
measured_value = results["results"][TASK][FILTER]
|
||||
expected_value = EXPECTED_VALUES.get(MODEL_NAME)
|
||||
|
||||
print(f"Measured accuracy value: {measured_value}\n")
|
||||
if expected_value is None:
|
||||
print(
|
||||
f"Warning: No expected value found for {MODEL_NAME}. "
|
||||
"Skipping accuracy check."
|
||||
)
|
||||
return
|
||||
|
||||
assert (
|
||||
measured_value - RTOL < expected_value
|
||||
and measured_value + RTOL > expected_value
|
||||
), f"Expected: {expected_value} | Measured: {measured_value}"
|
||||
@@ -0,0 +1,22 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Mooncake import canary for CUDA wheel selection."""
|
||||
|
||||
import importlib.metadata as metadata
|
||||
|
||||
|
||||
def test_mooncake_engine_imports():
|
||||
"""Fail here rather than as "Mooncake is not available" at engine startup.
|
||||
|
||||
The PyPI mooncake-transfer-engine wheel is built against CUDA 12, so on the
|
||||
CUDA 13 image install-kv-connectors.sh replaces it with the cuda13 variant.
|
||||
A mismatch surfaces as an ImportError on the compiled extension.
|
||||
"""
|
||||
for package_name in ("mooncake-transfer-engine", "mooncake-transfer-engine-cuda13"):
|
||||
try:
|
||||
version = metadata.version(package_name)
|
||||
except metadata.PackageNotFoundError:
|
||||
version = "not installed"
|
||||
print(f"{package_name}: {version}")
|
||||
|
||||
import mooncake.engine # noqa: F401
|
||||
Reference in New Issue
Block a user