[Bugfix][KV Connector] Retry Mooncake bootstrap registration on timeout (reopens #55763) (#58919)

This commit is contained in:
Chao Lei
2026-09-27 19:47:36 +00:00
committed by GitHub
parent 187c81b32d
commit 386ac2573e
5 changed files with 254 additions and 14 deletions
@@ -46,6 +46,13 @@ Now you can send requests to the proxy server through port 8000.
- For headless instances, must be the same as the master instance
- Each instance needs a unique port on its host; using the same port number across different hosts is fine
- `VLLM_MOONCAKE_CONNECTOR_TIMEOUT`: Per-request timeout (in seconds) for MooncakeConnector HTTP calls to the bootstrap server, including prefiller worker registration at startup. (Optional)
- Default: 30.0
- Global rank 0 hosts the bootstrap server in the same process that mounts the Mooncake transfer engine's host segment, so a large `MOONCAKE_GLOBAL_SEGMENT_SIZE` can delay responses for several seconds during startup
- Raise this if bootstrap server timeout warnings appear on hosts with very large host-memory segments
- Bootstrap registration uses up to 3 attempts internally, with exponential backoff capped at 10 seconds; connection errors and timeouts are both retried
- Once the attempts are exhausted the caller raises instead of blocking, so failures surface rather than hanging
- `WITH_NVIDIA_PEERMEM`: Selects how mooncake registers GPU memory for RDMA. Read by mooncake, not vLLM.
- Default: 1, which uses `ibv_reg_mr()` and requires the `nvidia-peermem` kernel module to be loaded
- Set to 0 to use the DMA-BUF path, which does not need that module. Required on hosts where `nvidia-peermem` is not loaded, such as GB200
@@ -3,10 +3,12 @@
import asyncio
import contextlib
import threading
import time
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
import torch
import zmq.asyncio
@@ -14,6 +16,7 @@ import zmq.asyncio
from tests.v1.attention.utils import dense_kv_cache_views
from vllm import envs
from vllm.config import set_current_vllm_config
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import mooncake_connector
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import (
KVConnectorRole,
MooncakeConnector,
@@ -33,6 +36,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector im
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import (
MooncakeBootstrapServer,
RegisterWorkerPayload,
)
from vllm.utils.network_utils import get_open_port
from vllm.v1.kv_cache_interface import (
@@ -538,6 +542,46 @@ def bootstrap_server():
server.shutdown()
@pytest.mark.asyncio
async def test_register_worker_recovers_from_slow_bootstrap_server(monkeypatch):
"""End-to-end: a real HTTP server that responds slower than the configured
timeout must be retried on the wire, not treated as fatal."""
original = MooncakeBootstrapServer.register_worker
state = {"done": False, "calls": 0}
async def slow_register(self, payload: RegisterWorkerPayload):
state["calls"] += 1
if not state["done"]:
state["done"] = True
# Record the worker first, then stall past the client timeout, so
# the retry exercises the duplicate-registration path.
response = await original(self, payload)
await asyncio.sleep(1.5)
return response
return await original(self, payload)
monkeypatch.setattr(MooncakeBootstrapServer, "register_worker", slow_register)
port = get_open_port()
monkeypatch.setenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", str(port))
monkeypatch.setenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "0.5")
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 5)
server = MooncakeBootstrapServer("127.0.0.1", port)
server.start()
try:
worker = _make_local_register_worker_stub()
await MooncakeConnectorWorker.register_worker_with_bootstrap(worker)
assert state["calls"] == 2
async with httpx.AsyncClient() as client:
response = await client.get(f"http://127.0.0.1:{port}/query")
assert response.status_code == 200
assert response.json()["0"]["engine_id"] == "eng-1"
finally:
server.shutdown()
@pytest.mark.asyncio
async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer):
"""Tests the bootstrap server's api for worker registration and querying.
@@ -589,9 +633,16 @@ async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer):
assert data["0"]["worker_addr"]["0"]["0"] == "tcp://1.1.1.1:1111"
assert data["0"]["worker_addr"]["0"]["1"] == "tcp://2.2.2.2:2222"
# Test failure: re-registering the same worker
# Re-registering the identical payload is idempotent.
async with httpx.AsyncClient() as client:
response = await client.post(f"{base_url}/register", json=payload1)
assert response.status_code == 200
assert response.json() == {"status": "ok"}
# Test failure: same ranks, conflicting address
payload1_conflict = dict(payload1, addr="tcp://9.9.9.9:9999")
async with httpx.AsyncClient() as client:
response = await client.post(f"{base_url}/register", json=payload1_conflict)
assert response.status_code == 400
assert "is already registered" in response.text
@@ -628,6 +679,137 @@ def _make_bootstrap_vllm_config(
)
def _make_local_register_worker_stub():
return SimpleNamespace(
vllm_config=SimpleNamespace(
parallel_config=SimpleNamespace(
local_engines_only=False,
data_parallel_rank_local=0,
data_parallel_index=0,
nnodes_within_dp=1,
master_addr="127.0.0.1",
data_parallel_master_ip="127.0.0.1",
)
),
hostname="127.0.0.1",
side_channel_port=1234,
engine_id="eng-1",
dp_rank=0,
tp_rank=0,
pp_rank=0,
)
def _make_register_worker_stub():
return SimpleNamespace(
vllm_config=_make_bootstrap_vllm_config(),
hostname="127.0.0.1",
side_channel_port=1234,
engine_id="eng-1",
dp_rank=0,
tp_rank=0,
pp_rank=0,
)
class _FlakyAsyncClient:
"""httpx.AsyncClient stub that read-times-out `failures` times first."""
def __init__(self, failures: int, calls: list[int], **kwargs):
self.failures = failures
self.calls = calls
async def __aenter__(self):
return self
async def __aexit__(self, *exc_info):
return False
async def post(self, url, json=None):
self.calls.append(1)
if len(self.calls) <= self.failures:
raise httpx.ReadTimeout("simulated slow bootstrap server")
response = MagicMock()
response.raise_for_status = MagicMock()
return response
@pytest.mark.asyncio
async def test_register_worker_retries_on_read_timeout(monkeypatch):
"""A slow rank-0 bootstrap response must be retried, not treated as fatal."""
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 5)
monkeypatch.setattr(asyncio, "sleep", AsyncMock())
calls: list[int] = []
monkeypatch.setattr(
httpx,
"AsyncClient",
lambda **kwargs: _FlakyAsyncClient(2, calls),
)
worker = _make_register_worker_stub()
await MooncakeConnectorWorker.register_worker_with_bootstrap(worker)
assert len(calls) == 3
@pytest.mark.asyncio
async def test_register_worker_raises_after_max_attempts(monkeypatch):
"""Terminal registration failure must raise, not loop forever."""
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 3)
monkeypatch.setattr(asyncio, "sleep", AsyncMock())
calls: list[int] = []
monkeypatch.setattr(
httpx,
"AsyncClient",
lambda **kwargs: _FlakyAsyncClient(99, calls),
)
worker = _make_register_worker_stub()
with pytest.raises(RuntimeError, match="after 3 attempts"):
await MooncakeConnectorWorker.register_worker_with_bootstrap(worker)
assert len(calls) == 3
@pytest.mark.asyncio
async def test_sender_listener_failure_propagates_to_caller(monkeypatch):
"""A terminal registration failure must surface via the Future rather than
leaving register_kv_caches blocked on its ready event."""
monkeypatch.setenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "0.1")
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 1)
loop = asyncio.new_event_loop()
thread = threading.Thread(target=loop.run_forever, daemon=True)
thread.start()
async def failing_listener(ready_event):
raise RuntimeError("simulated terminal registration failure")
try:
ready_event = threading.Event()
fut = asyncio.run_coroutine_threadsafe(failing_listener(ready_event), loop)
deadline = time.monotonic() + 10.0
raised = None
while not ready_event.wait(timeout=0.1):
if fut.done():
try:
fut.result()
except RuntimeError as e:
raised = e
break
assert time.monotonic() < deadline, "caller blocked past its deadline"
assert raised is not None
assert "simulated terminal registration failure" in str(raised)
finally:
loop.call_soon_threadsafe(loop.stop)
thread.join(timeout=5)
loop.close()
@pytest.mark.parametrize(
(
"tp_rank",
@@ -10,7 +10,7 @@ from collections.abc import Collection
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from enum import IntEnum
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, Final
import msgspec
import numpy as np
@@ -70,6 +70,8 @@ from vllm.v1.worker.utils import select_common_block_size
logger = init_logger(__name__)
_BOOTSTRAP_MAX_ATTEMPTS: Final[int] = 3
try:
from mooncake.engine import TransferEngine
except ImportError:
@@ -1147,24 +1149,44 @@ class MooncakeConnectorWorker:
pp_rank=self.pp_rank,
addr=worker_addr,
)
while True:
timeout = envs.VLLM_MOONCAKE_CONNECTOR_TIMEOUT
max_attempts = _BOOTSTRAP_MAX_ATTEMPTS
backoff = 1.0
for attempt in range(1, max_attempts + 1):
try:
async with httpx.AsyncClient() as client:
async with httpx.AsyncClient(timeout=timeout) as client:
response = await client.post(url, json=payload.model_dump())
response.raise_for_status()
logger.debug("Successfully registered with bootstrap server at %s", url)
break
except httpx.ConnectError:
# Bootstrap server not ready, wait for a while and retry.
await asyncio.sleep(1)
return
except (httpx.ConnectError, httpx.TimeoutException) as e:
# Rank 0 may be busy registering a large memory segment.
if attempt == max_attempts:
break
logger.warning(
"Bootstrap registration attempt %d/%d for %s failed with %s: %s. "
"Retrying in %.1fs.",
attempt,
max_attempts,
payload,
type(e).__name__,
e,
backoff,
)
await asyncio.sleep(backoff)
backoff = min(backoff * 2, 10.0)
except Exception as e:
err_msg = (
e.response.text if isinstance(e, httpx.HTTPStatusError) else str(e)
)
logger.error(
logger.exception(
"Error registering %s with bootstrap server: %s", payload, err_msg
)
raise e
raise
raise RuntimeError(
f"Failed to register {payload} with the Mooncake bootstrap server at "
f"{url} after {max_attempts} attempts."
)
async def _mooncake_sender_listener(self, ready_event: threading.Event):
"""Background thread that listens for Mooncake requests, dispatches them
@@ -1800,11 +1822,29 @@ class MooncakeConnectorWorker:
if self.is_kv_consumer:
return
# httpx applies the timeout to each phase (connect, read, write, pool)
# rather than to the whole request, so budget for all four.
register_timeout = envs.VLLM_MOONCAKE_CONNECTOR_TIMEOUT
max_attempts = _BOOTSTRAP_MAX_ATTEMPTS
backoff_total = sum(min(2.0**i, 10.0) for i in range(max_attempts - 1))
ready_timeout = max_attempts * 4 * register_timeout + backoff_total + 30.0
ready_event = threading.Event()
asyncio.run_coroutine_threadsafe(
fut = asyncio.run_coroutine_threadsafe(
self._mooncake_sender_listener(ready_event), self.sender_loop
)
ready_event.wait() # Wait for listener ZMQ socket to be ready.
deadline = time.monotonic() + ready_timeout
while not ready_event.wait(timeout=1.0):
if fut.done():
fut.result()
raise RuntimeError(
"Mooncake sender listener exited before becoming ready."
)
if time.monotonic() > deadline:
fut.cancel()
raise RuntimeError(
"Mooncake sender listener did not become ready within "
f"{ready_timeout:.0f}s."
)
async def fetch_finished_recving_reqs(self) -> set[ReqId]:
finished_recving_reqs = self.finished_recving_reqs
@@ -97,14 +97,19 @@ class MooncakeBootstrapServer:
dp_entry.worker_addr[payload.tp_rank] = {}
tp_entry = dp_entry.worker_addr[payload.tp_rank]
if payload.pp_rank in tp_entry:
existing = tp_entry.get(payload.pp_rank)
if existing is not None:
# A client timeout can fire after the server recorded the
# registration so an identical retry must not be an error.
if existing == payload.addr:
return {"status": "ok"}
raise HTTPException(
status_code=400,
detail=(
f"Worker with dp_rank={payload.dp_rank}, "
f"tp_rank={payload.tp_rank}, pp_rank={payload.pp_rank} "
f"is already registered at "
f"{tp_entry[payload.pp_rank]}, "
f"{existing}, "
f"but still want to register at {payload.addr}"
),
)
+6
View File
@@ -260,6 +260,7 @@ if TYPE_CHECKING:
VLLM_ROCM_QUICK_REDUCE_MIN_SIZE_BYTES_MB: int | None = None
VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB: int | None = None
VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT: int = 480
VLLM_MOONCAKE_CONNECTOR_TIMEOUT: float = 30.0
VLLM_ENABLE_CUDAGRAPH_GC: bool = False
VLLM_LOOPBACK_IP: str = ""
VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = True
@@ -1713,6 +1714,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_EC_SIDE_CHANNEL_PORT": lambda: int(
os.getenv("VLLM_EC_SIDE_CHANNEL_PORT", "5601")
),
# Per-request timeout (seconds) when a prefiller worker registers with
# the Mooncake bootstrap server.
"VLLM_MOONCAKE_CONNECTOR_TIMEOUT": lambda: float(
os.getenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "30.0")
),
# Port used for Mooncake handshake between remote agents.
"VLLM_MOONCAKE_BOOTSTRAP_PORT": lambda: int(
os.getenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", "8998")