diff --git a/docs/features/mooncake_connector_usage.md b/docs/features/mooncake_connector_usage.md index 6cce042fb6..7c5852a68b 100644 --- a/docs/features/mooncake_connector_usage.md +++ b/docs/features/mooncake_connector_usage.md @@ -46,6 +46,13 @@ Now you can send requests to the proxy server through port 8000. - For headless instances, must be the same as the master instance - Each instance needs a unique port on its host; using the same port number across different hosts is fine +- `VLLM_MOONCAKE_CONNECTOR_TIMEOUT`: Per-request timeout (in seconds) for MooncakeConnector HTTP calls to the bootstrap server, including prefiller worker registration at startup. (Optional) + - Default: 30.0 + - Global rank 0 hosts the bootstrap server in the same process that mounts the Mooncake transfer engine's host segment, so a large `MOONCAKE_GLOBAL_SEGMENT_SIZE` can delay responses for several seconds during startup + - Raise this if bootstrap server timeout warnings appear on hosts with very large host-memory segments + - Bootstrap registration uses up to 3 attempts internally, with exponential backoff capped at 10 seconds; connection errors and timeouts are both retried + - Once the attempts are exhausted the caller raises instead of blocking, so failures surface rather than hanging + - `WITH_NVIDIA_PEERMEM`: Selects how mooncake registers GPU memory for RDMA. Read by mooncake, not vLLM. - Default: 1, which uses `ibv_reg_mr()` and requires the `nvidia-peermem` kernel module to be loaded - Set to 0 to use the DMA-BUF path, which does not need that module. Required on hosts where `nvidia-peermem` is not loaded, such as GB200 diff --git a/tests/v1/kv_connector/unit/test_mooncake_connector.py b/tests/v1/kv_connector/unit/test_mooncake_connector.py index db9cf94171..5a5fffac3d 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_connector.py +++ b/tests/v1/kv_connector/unit/test_mooncake_connector.py @@ -3,10 +3,12 @@ import asyncio import contextlib +import threading import time from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest import torch import zmq.asyncio @@ -14,6 +16,7 @@ import zmq.asyncio from tests.v1.attention.utils import dense_kv_cache_views from vllm import envs from vllm.config import set_current_vllm_config +from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import mooncake_connector from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import ( KVConnectorRole, MooncakeConnector, @@ -33,6 +36,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector im ) from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import ( MooncakeBootstrapServer, + RegisterWorkerPayload, ) from vllm.utils.network_utils import get_open_port from vllm.v1.kv_cache_interface import ( @@ -538,6 +542,46 @@ def bootstrap_server(): server.shutdown() +@pytest.mark.asyncio +async def test_register_worker_recovers_from_slow_bootstrap_server(monkeypatch): + """End-to-end: a real HTTP server that responds slower than the configured + timeout must be retried on the wire, not treated as fatal.""" + original = MooncakeBootstrapServer.register_worker + state = {"done": False, "calls": 0} + + async def slow_register(self, payload: RegisterWorkerPayload): + state["calls"] += 1 + if not state["done"]: + state["done"] = True + # Record the worker first, then stall past the client timeout, so + # the retry exercises the duplicate-registration path. + response = await original(self, payload) + await asyncio.sleep(1.5) + return response + return await original(self, payload) + + monkeypatch.setattr(MooncakeBootstrapServer, "register_worker", slow_register) + + port = get_open_port() + monkeypatch.setenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", str(port)) + monkeypatch.setenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "0.5") + monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 5) + + server = MooncakeBootstrapServer("127.0.0.1", port) + server.start() + try: + worker = _make_local_register_worker_stub() + await MooncakeConnectorWorker.register_worker_with_bootstrap(worker) + assert state["calls"] == 2 + + async with httpx.AsyncClient() as client: + response = await client.get(f"http://127.0.0.1:{port}/query") + assert response.status_code == 200 + assert response.json()["0"]["engine_id"] == "eng-1" + finally: + server.shutdown() + + @pytest.mark.asyncio async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer): """Tests the bootstrap server's api for worker registration and querying. @@ -589,9 +633,16 @@ async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer): assert data["0"]["worker_addr"]["0"]["0"] == "tcp://1.1.1.1:1111" assert data["0"]["worker_addr"]["0"]["1"] == "tcp://2.2.2.2:2222" - # Test failure: re-registering the same worker + # Re-registering the identical payload is idempotent. async with httpx.AsyncClient() as client: response = await client.post(f"{base_url}/register", json=payload1) + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + + # Test failure: same ranks, conflicting address + payload1_conflict = dict(payload1, addr="tcp://9.9.9.9:9999") + async with httpx.AsyncClient() as client: + response = await client.post(f"{base_url}/register", json=payload1_conflict) assert response.status_code == 400 assert "is already registered" in response.text @@ -628,6 +679,137 @@ def _make_bootstrap_vllm_config( ) +def _make_local_register_worker_stub(): + return SimpleNamespace( + vllm_config=SimpleNamespace( + parallel_config=SimpleNamespace( + local_engines_only=False, + data_parallel_rank_local=0, + data_parallel_index=0, + nnodes_within_dp=1, + master_addr="127.0.0.1", + data_parallel_master_ip="127.0.0.1", + ) + ), + hostname="127.0.0.1", + side_channel_port=1234, + engine_id="eng-1", + dp_rank=0, + tp_rank=0, + pp_rank=0, + ) + + +def _make_register_worker_stub(): + return SimpleNamespace( + vllm_config=_make_bootstrap_vllm_config(), + hostname="127.0.0.1", + side_channel_port=1234, + engine_id="eng-1", + dp_rank=0, + tp_rank=0, + pp_rank=0, + ) + + +class _FlakyAsyncClient: + """httpx.AsyncClient stub that read-times-out `failures` times first.""" + + def __init__(self, failures: int, calls: list[int], **kwargs): + self.failures = failures + self.calls = calls + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc_info): + return False + + async def post(self, url, json=None): + self.calls.append(1) + if len(self.calls) <= self.failures: + raise httpx.ReadTimeout("simulated slow bootstrap server") + response = MagicMock() + response.raise_for_status = MagicMock() + return response + + +@pytest.mark.asyncio +async def test_register_worker_retries_on_read_timeout(monkeypatch): + """A slow rank-0 bootstrap response must be retried, not treated as fatal.""" + monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 5) + monkeypatch.setattr(asyncio, "sleep", AsyncMock()) + + calls: list[int] = [] + monkeypatch.setattr( + httpx, + "AsyncClient", + lambda **kwargs: _FlakyAsyncClient(2, calls), + ) + + worker = _make_register_worker_stub() + await MooncakeConnectorWorker.register_worker_with_bootstrap(worker) + + assert len(calls) == 3 + + +@pytest.mark.asyncio +async def test_register_worker_raises_after_max_attempts(monkeypatch): + """Terminal registration failure must raise, not loop forever.""" + monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 3) + monkeypatch.setattr(asyncio, "sleep", AsyncMock()) + + calls: list[int] = [] + monkeypatch.setattr( + httpx, + "AsyncClient", + lambda **kwargs: _FlakyAsyncClient(99, calls), + ) + + worker = _make_register_worker_stub() + with pytest.raises(RuntimeError, match="after 3 attempts"): + await MooncakeConnectorWorker.register_worker_with_bootstrap(worker) + + assert len(calls) == 3 + + +@pytest.mark.asyncio +async def test_sender_listener_failure_propagates_to_caller(monkeypatch): + """A terminal registration failure must surface via the Future rather than + leaving register_kv_caches blocked on its ready event.""" + monkeypatch.setenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "0.1") + monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 1) + + loop = asyncio.new_event_loop() + thread = threading.Thread(target=loop.run_forever, daemon=True) + thread.start() + + async def failing_listener(ready_event): + raise RuntimeError("simulated terminal registration failure") + + try: + ready_event = threading.Event() + fut = asyncio.run_coroutine_threadsafe(failing_listener(ready_event), loop) + + deadline = time.monotonic() + 10.0 + raised = None + while not ready_event.wait(timeout=0.1): + if fut.done(): + try: + fut.result() + except RuntimeError as e: + raised = e + break + assert time.monotonic() < deadline, "caller blocked past its deadline" + + assert raised is not None + assert "simulated terminal registration failure" in str(raised) + finally: + loop.call_soon_threadsafe(loop.stop) + thread.join(timeout=5) + loop.close() + + @pytest.mark.parametrize( ( "tp_rank", diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py index beabf157cc..5ef4f8f670 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py @@ -10,7 +10,7 @@ from collections.abc import Collection from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from enum import IntEnum -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Final import msgspec import numpy as np @@ -70,6 +70,8 @@ from vllm.v1.worker.utils import select_common_block_size logger = init_logger(__name__) +_BOOTSTRAP_MAX_ATTEMPTS: Final[int] = 3 + try: from mooncake.engine import TransferEngine except ImportError: @@ -1147,24 +1149,44 @@ class MooncakeConnectorWorker: pp_rank=self.pp_rank, addr=worker_addr, ) - while True: + timeout = envs.VLLM_MOONCAKE_CONNECTOR_TIMEOUT + max_attempts = _BOOTSTRAP_MAX_ATTEMPTS + backoff = 1.0 + for attempt in range(1, max_attempts + 1): try: - async with httpx.AsyncClient() as client: + async with httpx.AsyncClient(timeout=timeout) as client: response = await client.post(url, json=payload.model_dump()) response.raise_for_status() logger.debug("Successfully registered with bootstrap server at %s", url) - break - except httpx.ConnectError: - # Bootstrap server not ready, wait for a while and retry. - await asyncio.sleep(1) + return + except (httpx.ConnectError, httpx.TimeoutException) as e: + # Rank 0 may be busy registering a large memory segment. + if attempt == max_attempts: + break + logger.warning( + "Bootstrap registration attempt %d/%d for %s failed with %s: %s. " + "Retrying in %.1fs.", + attempt, + max_attempts, + payload, + type(e).__name__, + e, + backoff, + ) + await asyncio.sleep(backoff) + backoff = min(backoff * 2, 10.0) except Exception as e: err_msg = ( e.response.text if isinstance(e, httpx.HTTPStatusError) else str(e) ) - logger.error( + logger.exception( "Error registering %s with bootstrap server: %s", payload, err_msg ) - raise e + raise + raise RuntimeError( + f"Failed to register {payload} with the Mooncake bootstrap server at " + f"{url} after {max_attempts} attempts." + ) async def _mooncake_sender_listener(self, ready_event: threading.Event): """Background thread that listens for Mooncake requests, dispatches them @@ -1800,11 +1822,29 @@ class MooncakeConnectorWorker: if self.is_kv_consumer: return + # httpx applies the timeout to each phase (connect, read, write, pool) + # rather than to the whole request, so budget for all four. + register_timeout = envs.VLLM_MOONCAKE_CONNECTOR_TIMEOUT + max_attempts = _BOOTSTRAP_MAX_ATTEMPTS + backoff_total = sum(min(2.0**i, 10.0) for i in range(max_attempts - 1)) + ready_timeout = max_attempts * 4 * register_timeout + backoff_total + 30.0 ready_event = threading.Event() - asyncio.run_coroutine_threadsafe( + fut = asyncio.run_coroutine_threadsafe( self._mooncake_sender_listener(ready_event), self.sender_loop ) - ready_event.wait() # Wait for listener ZMQ socket to be ready. + deadline = time.monotonic() + ready_timeout + while not ready_event.wait(timeout=1.0): + if fut.done(): + fut.result() + raise RuntimeError( + "Mooncake sender listener exited before becoming ready." + ) + if time.monotonic() > deadline: + fut.cancel() + raise RuntimeError( + "Mooncake sender listener did not become ready within " + f"{ready_timeout:.0f}s." + ) async def fetch_finished_recving_reqs(self) -> set[ReqId]: finished_recving_reqs = self.finished_recving_reqs diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_utils.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_utils.py index df47f9fdb7..dfe0641b36 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_utils.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_utils.py @@ -97,14 +97,19 @@ class MooncakeBootstrapServer: dp_entry.worker_addr[payload.tp_rank] = {} tp_entry = dp_entry.worker_addr[payload.tp_rank] - if payload.pp_rank in tp_entry: + existing = tp_entry.get(payload.pp_rank) + if existing is not None: + # A client timeout can fire after the server recorded the + # registration so an identical retry must not be an error. + if existing == payload.addr: + return {"status": "ok"} raise HTTPException( status_code=400, detail=( f"Worker with dp_rank={payload.dp_rank}, " f"tp_rank={payload.tp_rank}, pp_rank={payload.pp_rank} " f"is already registered at " - f"{tp_entry[payload.pp_rank]}, " + f"{existing}, " f"but still want to register at {payload.addr}" ), ) diff --git a/vllm/envs.py b/vllm/envs.py index b0d72a71db..f663414a91 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -260,6 +260,7 @@ if TYPE_CHECKING: VLLM_ROCM_QUICK_REDUCE_MIN_SIZE_BYTES_MB: int | None = None VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB: int | None = None VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT: int = 480 + VLLM_MOONCAKE_CONNECTOR_TIMEOUT: float = 30.0 VLLM_ENABLE_CUDAGRAPH_GC: bool = False VLLM_LOOPBACK_IP: str = "" VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = True @@ -1713,6 +1714,11 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_EC_SIDE_CHANNEL_PORT": lambda: int( os.getenv("VLLM_EC_SIDE_CHANNEL_PORT", "5601") ), + # Per-request timeout (seconds) when a prefiller worker registers with + # the Mooncake bootstrap server. + "VLLM_MOONCAKE_CONNECTOR_TIMEOUT": lambda: float( + os.getenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "30.0") + ), # Port used for Mooncake handshake between remote agents. "VLLM_MOONCAKE_BOOTSTRAP_PORT": lambda: int( os.getenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", "8998")