diff --git a/.buildkite/scripts/install-kv-connectors.sh b/.buildkite/scripts/install-kv-connectors.sh index b1e024709e..ef47c3625f 100755 --- a/.buildkite/scripts/install-kv-connectors.sh +++ b/.buildkite/scripts/install-kv-connectors.sh @@ -13,7 +13,7 @@ REQUIREMENTS_FILE="${KV_CONNECTORS_REQUIREMENTS:-/vllm-workspace/requirements/kv uv pip install --system -r "${REQUIREMENTS_FILE}" -NIXL_METADATA=$(python3 - <<'PY' +KV_METADATA=$(python3 - <<'PY' import importlib.metadata as metadata import torch @@ -22,10 +22,16 @@ cuda_version = torch.version.cuda if cuda_version is None: raise SystemExit("torch.version.cuda is not set") -print(cuda_version.split(".", 1)[0], metadata.version("nixl")) +try: + mooncake_version = metadata.version("mooncake-transfer-engine") +except metadata.PackageNotFoundError: + mooncake_version = "" + +print(cuda_version.split(".", 1)[0], metadata.version("nixl"), mooncake_version) PY ) -read -r CUDA_MAJOR NIXL_VERSION <<<"${NIXL_METADATA}" +read -r CUDA_MAJOR NIXL_VERSION MOONCAKE_VERSION <<<"${KV_METADATA}" +MOONCAKE_VERSION="${MOONCAKE_VERSION:-}" # nixl>=1.1.0 can install multiple CUDA wheel variants. Keep only the variant # matching this CI image so nixl_ep_cpp links against the available libcudart. @@ -42,3 +48,13 @@ for package_name in ("nixl", "nixl-cu12", "nixl-cu13"): version = "not installed" print(f"{package_name}: {version}") PY + +# The default mooncake-transfer-engine PyPI wheel is built against CUDA 12; its +# engine.so links libcudart.so.12, absent from the CUDA 13 runtime image. On a +# CUDA 13 image, swap it for the cuda13 variant (same version), which links +# libcudart.so.13. Both expose the `mooncake` package, so uninstall the CUDA 12 +# build first to avoid a clash. +if [ "${CUDA_MAJOR}" = "13" ] && [ -n "${MOONCAKE_VERSION}" ]; then + uv pip uninstall --system mooncake-transfer-engine 2>/dev/null || true + uv pip install --system "mooncake-transfer-engine-cuda13==${MOONCAKE_VERSION}" +fi diff --git a/.buildkite/test_areas/disaggregated_mooncake.yaml b/.buildkite/test_areas/disaggregated_mooncake.yaml new file mode 100644 index 0000000000..92a9e625d1 --- /dev/null +++ b/.buildkite/test_areas/disaggregated_mooncake.yaml @@ -0,0 +1,21 @@ +group: Disaggregated Mooncake +depends_on: + - image-build +steps: +- label: Distributed MooncakeConnector PD accuracy (4 GPUs) + key: distributed-mooncakeconnector-pd-accuracy-4-gpus + timeout_in_minutes: 10 + working_dir: "/vllm-workspace/tests" + # Runs on b200-k8s because it is the only NVIDIA pool that can carry mooncake + # transfers: its pods are the only ones granted IPC_LOCK, which RDMA needs to + # pin memory, and the only ones using host networking. Elsewhere topology + # discovery finds no usable HCA and the tcp fallback then advertises the + # docker bridge address, so every transfer fails to connect. + device: b200-k8s + num_devices: 4 + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/mooncake/ + - tests/v1/kv_connector/mooncake_integration/ + commands: + - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh + - bash v1/kv_connector/mooncake_integration/config_sweep_accuracy_test.sh diff --git a/docs/features/mooncake_connector_usage.md b/docs/features/mooncake_connector_usage.md index 5fef3bf65a..e6900e9877 100644 --- a/docs/features/mooncake_connector_usage.md +++ b/docs/features/mooncake_connector_usage.md @@ -60,6 +60,7 @@ Now you can send requests to the proxy server through port 8000. - **num_workers**: Size of thread pool for one prefiller worker to transfer KV caches by mooncake. (default 10) - **mooncake_protocol**: Mooncake connector protocol. (default "rdma") +- **device_name**: Comma-separated whitelist of RDMA devices (e.g. `"mlx5_0,mlx5_1"`) to restrict topology discovery to. Empty discovers every device. Useful on hosts exposing a mix of InfiniBand and RoCE ports, where both peers must settle on the same link layer. ## Example Scripts/Code diff --git a/requirements/kv_connectors.txt b/requirements/kv_connectors.txt index c61ac49539..78b1964c3f 100644 --- a/requirements/kv_connectors.txt +++ b/requirements/kv_connectors.txt @@ -3,4 +3,6 @@ lmcache >= 0.3.9 # until a fixed newer release is verified for runtime images. cupy-cuda13x < 14.1.0 nixl == 1.3.1 -mooncake-transfer-engine >= 0.3.8 +# CUDA 12 build. On the CUDA 13 image install-kv-connectors.sh swaps this for +# the mooncake-transfer-engine-cuda13 variant of the same version. +mooncake-transfer-engine >= 0.3.12 diff --git a/tests/v1/kv_connector/mooncake_integration/__init__.py b/tests/v1/kv_connector/mooncake_integration/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/v1/kv_connector/mooncake_integration/config_sweep_accuracy_test.sh b/tests/v1/kv_connector/mooncake_integration/config_sweep_accuracy_test.sh new file mode 100755 index 0000000000..48c446e019 --- /dev/null +++ b/tests/v1/kv_connector/mooncake_integration/config_sweep_accuracy_test.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Utility to run Mooncake P/D integration tests sequentially with varying TP +# configurations. +SCRIPT="v1/kv_connector/mooncake_integration/run_accuracy_test.sh" +IMPORT_CANARY="v1/kv_connector/mooncake_integration/test_mooncake_imports.py" + +echo "=== Running Mooncake import canary ===" +python3 -m pytest -s -x "${IMPORT_CANARY}" + +# Symmetric TP configurations (default model: Qwen/Qwen3-0.6B). +tp_configs=( + "GPU_MEMORY_UTILIZATION=0.6 PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=1" + "GPU_MEMORY_UTILIZATION=0.6 PREFILLER_TP_SIZE=2 DECODER_TP_SIZE=2" +) + +configs=("${tp_configs[@]}") + +run_tests() { + local label=$1 + local extra_args=$2 + + echo "=== Running tests (${label}) ===" + for cfg in "${configs[@]}"; do + local -a cfg_parts extra_args_parts + read -r -a cfg_parts <<< "$cfg" + read -r -a extra_args_parts <<< "$extra_args" + + echo "-> Running with ${cfg} ${extra_args:+and ${extra_args}}" + # Use 'env' to safely set variables without eval + # keep argv splitting safe and SC2086-clean via arrays. + if ! env "${cfg_parts[@]}" bash "${SCRIPT}" "${extra_args_parts[@]}"; then + echo "❌ Test failed for config: ${cfg} ${extra_args:+(${extra_args})}" + exit 1 + fi + done + echo "✅ All ${label} tests passed!" +} + +# Set backend +label="default backend" +cmdline_args="" +if [[ -n "${ATTENTION_BACKEND:-}" ]]; then + echo "ATTENTION_BACKEND is set, running with --attention-backend ${ATTENTION_BACKEND}" + label="${ATTENTION_BACKEND} backend" + cmdline_args=" --attention-backend ${ATTENTION_BACKEND} " +else + echo "running with default attention backend" +fi + +# Run tests +run_tests "${label}" "${cmdline_args}" diff --git a/tests/v1/kv_connector/mooncake_integration/run_accuracy_test.sh b/tests/v1/kv_connector/mooncake_integration/run_accuracy_test.sh new file mode 100755 index 0000000000..9b3d129804 --- /dev/null +++ b/tests/v1/kv_connector/mooncake_integration/run_accuracy_test.sh @@ -0,0 +1,227 @@ +#!/bin/bash +set -xe + +# Parse command line arguments +ATTENTION_BACKEND="" # Default to empty (use vllm default) + +while [[ $# -gt 0 ]]; do + case $1 in + --attention-backend) + ATTENTION_BACKEND="$2" + shift 2 + ;; + *) + echo "Unknown option $1" + echo "Usage: $0 [--attention-backend ]" + exit 1 + ;; + esac +done + +if [[ -n "$ATTENTION_BACKEND" ]]; then + echo "Using attention backend: $ATTENTION_BACKEND" +fi +if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then + echo "vLLM serve extra args: $VLLM_SERVE_EXTRA_ARGS" +fi + +# Mooncake discovers peers via its bootstrap server, so no side-channel +# or layout env vars are needed. + +# Comma-separated whitelist restricting RDMA topology discovery to named HCAs. +# This default is specific to the b200-k8s CI pool: those nodes expose four +# InfiniBand HCAs that have no network device next to a single RoCE port +# (mlx5_12). With all five visible the two sides can settle on different link +# layers, and the QP handshake then fails to reach RTR. Set to a different NIC +# on other hardware, or to the empty string to discover every device. +MOONCAKE_DEVICE_NAME=${MOONCAKE_DEVICE_NAME-mlx5_12} +kv_config() { + printf '{"kv_connector":"MooncakeConnector","kv_role":"%s",' "$1" + printf '"kv_connector_extra_config":{"device_name":"%s"}}' "${MOONCAKE_DEVICE_NAME}" +} +KV_CONFIG_P=$(kv_config kv_producer) +KV_CONFIG_D=$(kv_config kv_consumer) + +# Models to run +MODEL_NAME=${MODEL_NAME:-} +if [[ -n "$MODEL_NAME" ]]; then + MODEL="$MODEL_NAME" +else + MODEL="Qwen/Qwen3-0.6B" +fi + +# Number of prefill and decode instances to create +NUM_PREFILL_INSTANCES=${NUM_PREFILL_INSTANCES:-1} # Default to 1 +NUM_DECODE_INSTANCES=${NUM_DECODE_INSTANCES:-1} # Default to 1 +PREFILLER_TP_SIZE=${PREFILLER_TP_SIZE:-1} +DECODER_TP_SIZE=${DECODER_TP_SIZE:-1} +GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.2} +BLOCK_SIZE=${BLOCK_SIZE:-128} +# Comma-separated extra args for vllm serve (e.g. --max-model-len,2048) +VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-} + +# Resolve the repository root from the script location instead of `.git`. +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}" + +SMI_BIN=$(which nvidia-smi || which rocm-smi || echo "") + +# Trap the SIGINT signal (triggered by Ctrl+C) +trap 'kill $(jobs -pr)' SIGINT SIGTERM EXIT + +# Waits for vLLM to start. +wait_for_server() { + local port=$1 + timeout 1200 bash -c " + until curl -s localhost:${port}/v1/completions > /dev/null; do + sleep 1 + done" && return 0 || return 1 +} + +# Function to clean up previous instances +cleanup_instances() { + echo "Cleaning up any running vLLM instances..." + pkill -f "vllm serve" || true + sleep 2 +} + +get_num_gpus() { + if [[ "$SMI_BIN" == *"nvidia"* ]]; then + $SMI_BIN --query-gpu=name --format=csv,noheader | wc -l + elif [[ "$SMI_BIN" == *"rocm"* ]]; then + $SMI_BIN -l | grep -c GPU + else + # works for non-cuda platforms, + # assuming at least 1 device and + # let system to decide which card to use + echo "1" + fi +} + +# Function to run tests for a specific model +run_tests_for_model() { + local model_name=$1 + echo "================================" + echo "Testing model: $model_name" + echo "================================" + + # Proxy args: repeated --prefill URL BOOTSTRAP_PORT and --decode URL. + PROXY_ARGS=() + + # Start prefill instances + for i in $(seq 0 $((NUM_PREFILL_INSTANCES-1))); do + # Calculate GPU ID - we'll distribute across available GPUs + GPU_ID=$((i % $(get_num_gpus))) + NEXT_GPU=${GPU_ID} + # If PREFILLER_TP_SIZE is more than 1 + for (( j=1; j < PREFILLER_TP_SIZE; j++ )); do + NEXT_GPU=$(((GPU_ID + j) % $(get_num_gpus))) + GPU_ID="${GPU_ID},${NEXT_GPU}" + done + + # Calculate port number (base port + instance number) + PORT=$((8100 + i)) + # Bootstrap server port for this prefiller. Avoid clash across instances. + BOOTSTRAP_PORT=$((8998 + i)) + + echo "Starting prefill instance $i on GPU $GPU_ID, port $PORT, bootstrap $BOOTSTRAP_PORT" + + BASE_CMD="CUDA_VISIBLE_DEVICES=$GPU_ID \ + VLLM_MOONCAKE_BOOTSTRAP_PORT=$BOOTSTRAP_PORT \ + vllm serve $model_name \ + --port $PORT \ + --enforce-eager \ + --block-size ${BLOCK_SIZE} \ + --gpu-memory-utilization $GPU_MEMORY_UTILIZATION \ + --tensor-parallel-size $PREFILLER_TP_SIZE \ + --kv-transfer-config '$KV_CONFIG_P'" + if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then + IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS" + for arg in "${extra_args[@]}"; do + BASE_CMD="${BASE_CMD} $arg" + done + fi + + # Add attention backend config if specified + if [[ -n "$ATTENTION_BACKEND" ]]; then + BASE_CMD="${BASE_CMD} --attention-backend=$ATTENTION_BACKEND" + fi + + eval "$BASE_CMD &" + + PROXY_ARGS+=(--prefill "http://localhost:${PORT}" "$BOOTSTRAP_PORT") + PREFILL_PORTS+=("$PORT") + done + + # Start decode instances + for i in $(seq 0 $((NUM_DECODE_INSTANCES-1))); do + # Calculate GPU ID - distribute starting from after prefill GPUs + GPU_ID=$(((i + NEXT_GPU + 1) % $(get_num_gpus))) + # If DECODER_TP_SIZE is more than 1 + for (( j=1; j < DECODER_TP_SIZE; j++ )); do + NEXT_GPU=$(((GPU_ID + j) % $(get_num_gpus))) + GPU_ID="${GPU_ID},${NEXT_GPU}" + done + # Calculate port number (base port + instance number) + PORT=$((8200 + i)) + + echo "Starting decode instance $i on GPU $GPU_ID, port $PORT" + + BASE_CMD="CUDA_VISIBLE_DEVICES=$GPU_ID \ + vllm serve $model_name \ + --port $PORT \ + --enforce-eager \ + --block-size ${BLOCK_SIZE} \ + --gpu-memory-utilization $GPU_MEMORY_UTILIZATION \ + --tensor-parallel-size $DECODER_TP_SIZE \ + --kv-transfer-config '$KV_CONFIG_D'" + if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then + IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS" + for arg in "${extra_args[@]}"; do + BASE_CMD="${BASE_CMD} $arg" + done + fi + + # Add attention backend config if specified + if [[ -n "$ATTENTION_BACKEND" ]]; then + BASE_CMD="${BASE_CMD} --attention-backend=$ATTENTION_BACKEND" + fi + + eval "$BASE_CMD &" + + PROXY_ARGS+=(--decode "http://localhost:${PORT}") + DECODE_PORTS+=("$PORT") + done + + # Wait for all instances to start + for PORT in "${PREFILL_PORTS[@]}"; do + echo "Waiting for prefill instance on port $PORT to start..." + wait_for_server "$PORT" + done + + for PORT in "${DECODE_PORTS[@]}"; do + echo "Waiting for decode instance on port $PORT to start..." + wait_for_server "$PORT" + done + + # Start the proxy server + echo "Starting proxy server on port 8192" + python3 "${GIT_ROOT}/examples/disaggregated/mooncake_connector/mooncake_connector_proxy.py" \ + --port 8192 "${PROXY_ARGS[@]}" & + + # Wait for the proxy to start + sleep 5 + + # Run lm eval for this model + echo "Running tests for $model_name" + TEST_MODEL=$model_name python3 -m pytest -s -x \ + "${GIT_ROOT}/tests/v1/kv_connector/mooncake_integration/test_accuracy.py" + + # Clean up before running next model + cleanup_instances + sleep 3 +} + +run_tests_for_model "$MODEL" + +echo "All tests completed!" diff --git a/tests/v1/kv_connector/mooncake_integration/test_accuracy.py b/tests/v1/kv_connector/mooncake_integration/test_accuracy.py new file mode 100644 index 0000000000..25033399e7 --- /dev/null +++ b/tests/v1/kv_connector/mooncake_integration/test_accuracy.py @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import os + +import lm_eval +import openai + +BASE_URL = "http://localhost:8192/v1" +NUM_CONCURRENT = 100 +TASK = "gsm8k" +FILTER = "exact_match,strict-match" +RTOL = 0.05 + +# Model-specific expected values +EXPECTED_VALUES = { + "Qwen/Qwen3-0.6B": 0.41, +} + +SIMPLE_PROMPT = ( + "The best part about working on vLLM is that I got to meet so many people across " + "various different organizations like UCB, Google, and Meta which means", +) + +# Get model name from environment variable +MODEL_NAME = os.environ.get("TEST_MODEL", "Qwen/Qwen3-0.6B") + + +def run_simple_prompt(): + client = openai.OpenAI(api_key="EMPTY", base_url=BASE_URL) + completion = client.completions.create(model=MODEL_NAME, prompt=SIMPLE_PROMPT) + + print("-" * 50) + print(f"Completion results for {MODEL_NAME}:") + print(completion) + print("-" * 50) + + +def test_accuracy(): + """Run the end to end accuracy test.""" + run_simple_prompt() + + model_args = ( + f"model={MODEL_NAME}," + f"base_url={BASE_URL}/completions," + f"num_concurrent={NUM_CONCURRENT},tokenized_requests=False" + ) + results = lm_eval.simple_evaluate( + model="local-completions", + model_args=model_args, + tasks=TASK, + ) + + measured_value = results["results"][TASK][FILTER] + expected_value = EXPECTED_VALUES.get(MODEL_NAME) + + print(f"Measured accuracy value: {measured_value}\n") + if expected_value is None: + print( + f"Warning: No expected value found for {MODEL_NAME}. " + "Skipping accuracy check." + ) + return + + assert ( + measured_value - RTOL < expected_value + and measured_value + RTOL > expected_value + ), f"Expected: {expected_value} | Measured: {measured_value}" diff --git a/tests/v1/kv_connector/mooncake_integration/test_mooncake_imports.py b/tests/v1/kv_connector/mooncake_integration/test_mooncake_imports.py new file mode 100644 index 0000000000..f339da2b92 --- /dev/null +++ b/tests/v1/kv_connector/mooncake_integration/test_mooncake_imports.py @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Mooncake import canary for CUDA wheel selection.""" + +import importlib.metadata as metadata + + +def test_mooncake_engine_imports(): + """Fail here rather than as "Mooncake is not available" at engine startup. + + The PyPI mooncake-transfer-engine wheel is built against CUDA 12, so on the + CUDA 13 image install-kv-connectors.sh replaces it with the cuda13 variant. + A mismatch surfaces as an ImportError on the compiled extension. + """ + for package_name in ("mooncake-transfer-engine", "mooncake-transfer-engine-cuda13"): + try: + version = metadata.version(package_name) + except metadata.PackageNotFoundError: + version = "not installed" + print(f"{package_name}: {version}") + + import mooncake.engine # noqa: F401