[KV Connector] Support sleep mode with MooncakeConnector over RDMA (#59625)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Dakai An <dakaian108@gmail.com>
Co-authored-by: Dakai An <77474977+andakai@users.noreply.github.com>
This commit is contained in:
aoshen02
2026-10-14 04:18:47 +00:00
committed by GitHub
co-authored by Claude Fable 5.1 Dakai An Dakai An
parent f4b8d43a68
commit 3ccc0a90b6
14 changed files with 205 additions and 8 deletions
@@ -13,6 +13,7 @@ python3 -m pytest -s -x "${IMPORT_CANARY}"
tp_configs=(
"GPU_MEMORY_UTILIZATION=0.6 PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=1"
"GPU_MEMORY_UTILIZATION=0.6 PREFILLER_TP_SIZE=2 DECODER_TP_SIZE=2"
"GPU_MEMORY_UTILIZATION=0.6 PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=1 SLEEP_WAKE=1"
)
configs=("${tp_configs[@]}")
@@ -59,6 +59,12 @@ GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.2}
BLOCK_SIZE=${BLOCK_SIZE:-128}
# Comma-separated extra args for vllm serve (e.g. --max-model-len,2048)
VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-}
# Set to 1 to put every instance to sleep and wake it up before the eval.
SLEEP_WAKE=${SLEEP_WAKE:-0}
if [[ "$SLEEP_WAKE" == "1" ]]; then
export VLLM_SERVER_DEV_MODE=1
VLLM_SERVE_EXTRA_ARGS="${VLLM_SERVE_EXTRA_ARGS:+${VLLM_SERVE_EXTRA_ARGS},}--enable-sleep-mode"
fi
# Resolve the repository root from the script location instead of `.git`.
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
@@ -226,9 +232,17 @@ run_tests_for_model() {
# Wait for the proxy to start
sleep 5
SLEEP_WAKE_URLS=""
if [[ "$SLEEP_WAKE" == "1" ]]; then
for PORT in "${PREFILL_PORTS[@]}" "${DECODE_PORTS[@]}"; do
SLEEP_WAKE_URLS="${SLEEP_WAKE_URLS:+${SLEEP_WAKE_URLS},}http://localhost:${PORT}"
done
fi
# Run lm eval for this model
echo "Running tests for $model_name"
TEST_MODEL=$model_name PROXY_PORT=$PROXY_PORT python3 -m pytest -s -x \
TEST_MODEL=$model_name PROXY_PORT=$PROXY_PORT SLEEP_WAKE_URLS=$SLEEP_WAKE_URLS \
python3 -m pytest -s -x \
"${GIT_ROOT}/tests/v1/kv_connector/mooncake_integration/test_accuracy.py"
# Clean up before running next model
@@ -4,8 +4,13 @@ import os
import lm_eval
import openai
import requests
BASE_URL = f"http://localhost:{os.environ.get('PROXY_PORT', 8192)}/v1"
# Comma-separated server URLs to sleep and wake up before the eval.
SLEEP_WAKE_URLS = [
url for url in os.environ.get("SLEEP_WAKE_URLS", "").split(",") if url
]
NUM_CONCURRENT = 100
TASK = "gsm8k"
FILTER = "exact_match,strict-match"
@@ -35,9 +40,18 @@ def run_simple_prompt():
print("-" * 50)
def sleep_and_wake_up(urls: list[str]):
for url in urls:
requests.post(f"{url}/sleep", params={"level": 1}).raise_for_status()
for url in urls:
requests.post(f"{url}/wake_up").raise_for_status()
def test_accuracy():
"""Run the end to end accuracy test."""
run_simple_prompt()
if SLEEP_WAKE_URLS:
sleep_and_wake_up(SLEEP_WAKE_URLS)
model_args = (
f"model={MODEL_NAME},"
+28 -2
View File
@@ -101,6 +101,7 @@ def test_kv_connector(
def _build_config(
*,
kv_connector: str | None,
kv_connector_extra_config: dict | None = None,
enable_sleep_mode: bool = False,
enable_cumem_allocator: bool = False,
) -> VllmConfig:
@@ -109,7 +110,11 @@ def _build_config(
from types import SimpleNamespace
kv_transfer_config = (
KVTransferConfig(kv_connector=kv_connector, kv_role="kv_both")
KVTransferConfig(
kv_connector=kv_connector,
kv_role="kv_both",
kv_connector_extra_config=kv_connector_extra_config or {},
)
if kv_connector is not None
else None
)
@@ -141,7 +146,7 @@ def test_kv_connector_allows_expandable_segments_with_sleep_mode(monkeypatch):
"""Sleep mode routes KV allocations through CuMemAllocator's pool, which
auto-disables expandable_segments (see #40812)."""
monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
_build_config(kv_connector="NixlConnector", enable_sleep_mode=True)
_build_config(kv_connector="MooncakeConnector", enable_sleep_mode=True)
def test_kv_connector_allows_expandable_segments_with_cumem_allocator(
@@ -152,6 +157,27 @@ def test_kv_connector_allows_expandable_segments_with_cumem_allocator(
_build_config(kv_connector="NixlConnector", enable_cumem_allocator=True)
@pytest.mark.parametrize(
("kv_connector", "extra_config", "env"),
[
("NixlConnector", {}, None),
("MooncakeConnector", {"mooncake_protocol": "tcp"}, None),
("MooncakeConnector", {}, "MC_USE_TENT"),
],
)
def test_sleep_mode_refuses_unsupported_kv_connector(
monkeypatch, kv_connector, extra_config, env
):
if env:
monkeypatch.setenv(env, "1")
with pytest.raises(ValueError, match=f"^{kv_connector} does not support sleep"):
_build_config(
kv_connector=kv_connector,
kv_connector_extra_config=extra_config,
enable_sleep_mode=True,
)
def test_kv_connector_allows_other_alloc_conf(monkeypatch):
"""Other PYTORCH_CUDA_ALLOC_CONF values must not be rejected."""
monkeypatch.setenv(
@@ -232,6 +232,9 @@ class FakeMooncakeWrapper:
def batch_register_memory(self, buffer_addresses, capacities) -> int:
return 0
def batch_unregister_memory(self, buffer_addresses) -> int:
return 0
def test_align_transfer_regions_uses_layer_name_occurrences():
"""Repeated layer names should align by occurrence order."""
@@ -2574,3 +2577,29 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size):
prefill_worker.sender_loop = origin_sender_loop
prefill_worker.shutdown()
def test_sleep_unregisters_and_restores_kv_caches():
spec = FullAttentionSpec(
block_size=maybe_update_block_size(16),
num_kv_heads=4,
head_size=64,
dtype=torch.float16,
)
layer_names = ["model.layers.0.self_attn", "model.layers.1.self_attn"]
with mooncake_register_worker(_make_test_kv_cache_config()) as (worker, reg):
for layer_name in layer_names:
worker._layer_specs[layer_name] = spec
raw = torch.zeros(2 * 2 * spec.page_size_bytes, dtype=torch.int8)
views = dense_kv_cache_views(raw, spec, 2, 2, KVCacheLayout.LBHNC)
worker.register_kv_caches(dict(zip(layer_names, views)))
reg.reset_mock()
with patch.object(
worker.engine, "batch_unregister_memory", return_value=0
) as unreg:
worker.unregister_kv_caches()
worker.unregister_kv_caches()
unreg.assert_called_once_with([raw.data_ptr()])
worker.restore_kv_caches()
worker.restore_kv_caches()
reg.assert_called_once_with([raw.data_ptr()], [raw.nbytes])