[Bugfix][KV Connector] Retry Mooncake bootstrap registration on timeout (reopens #55763) (#58919)

This commit is contained in:
Chao Lei
2026-09-27 19:47:36 +00:00
committed by GitHub
parent 187c81b32d
commit 386ac2573e
5 changed files with 254 additions and 14 deletions
@@ -10,7 +10,7 @@ from collections.abc import Collection
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from enum import IntEnum
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, Final
import msgspec
import numpy as np
@@ -70,6 +70,8 @@ from vllm.v1.worker.utils import select_common_block_size
logger = init_logger(__name__)
_BOOTSTRAP_MAX_ATTEMPTS: Final[int] = 3
try:
from mooncake.engine import TransferEngine
except ImportError:
@@ -1147,24 +1149,44 @@ class MooncakeConnectorWorker:
pp_rank=self.pp_rank,
addr=worker_addr,
)
while True:
timeout = envs.VLLM_MOONCAKE_CONNECTOR_TIMEOUT
max_attempts = _BOOTSTRAP_MAX_ATTEMPTS
backoff = 1.0
for attempt in range(1, max_attempts + 1):
try:
async with httpx.AsyncClient() as client:
async with httpx.AsyncClient(timeout=timeout) as client:
response = await client.post(url, json=payload.model_dump())
response.raise_for_status()
logger.debug("Successfully registered with bootstrap server at %s", url)
break
except httpx.ConnectError:
# Bootstrap server not ready, wait for a while and retry.
await asyncio.sleep(1)
return
except (httpx.ConnectError, httpx.TimeoutException) as e:
# Rank 0 may be busy registering a large memory segment.
if attempt == max_attempts:
break
logger.warning(
"Bootstrap registration attempt %d/%d for %s failed with %s: %s. "
"Retrying in %.1fs.",
attempt,
max_attempts,
payload,
type(e).__name__,
e,
backoff,
)
await asyncio.sleep(backoff)
backoff = min(backoff * 2, 10.0)
except Exception as e:
err_msg = (
e.response.text if isinstance(e, httpx.HTTPStatusError) else str(e)
)
logger.error(
logger.exception(
"Error registering %s with bootstrap server: %s", payload, err_msg
)
raise e
raise
raise RuntimeError(
f"Failed to register {payload} with the Mooncake bootstrap server at "
f"{url} after {max_attempts} attempts."
)
async def _mooncake_sender_listener(self, ready_event: threading.Event):
"""Background thread that listens for Mooncake requests, dispatches them
@@ -1800,11 +1822,29 @@ class MooncakeConnectorWorker:
if self.is_kv_consumer:
return
# httpx applies the timeout to each phase (connect, read, write, pool)
# rather than to the whole request, so budget for all four.
register_timeout = envs.VLLM_MOONCAKE_CONNECTOR_TIMEOUT
max_attempts = _BOOTSTRAP_MAX_ATTEMPTS
backoff_total = sum(min(2.0**i, 10.0) for i in range(max_attempts - 1))
ready_timeout = max_attempts * 4 * register_timeout + backoff_total + 30.0
ready_event = threading.Event()
asyncio.run_coroutine_threadsafe(
fut = asyncio.run_coroutine_threadsafe(
self._mooncake_sender_listener(ready_event), self.sender_loop
)
ready_event.wait() # Wait for listener ZMQ socket to be ready.
deadline = time.monotonic() + ready_timeout
while not ready_event.wait(timeout=1.0):
if fut.done():
fut.result()
raise RuntimeError(
"Mooncake sender listener exited before becoming ready."
)
if time.monotonic() > deadline:
fut.cancel()
raise RuntimeError(
"Mooncake sender listener did not become ready within "
f"{ready_timeout:.0f}s."
)
async def fetch_finished_recving_reqs(self) -> set[ReqId]:
finished_recving_reqs = self.finished_recving_reqs
@@ -97,14 +97,19 @@ class MooncakeBootstrapServer:
dp_entry.worker_addr[payload.tp_rank] = {}
tp_entry = dp_entry.worker_addr[payload.tp_rank]
if payload.pp_rank in tp_entry:
existing = tp_entry.get(payload.pp_rank)
if existing is not None:
# A client timeout can fire after the server recorded the
# registration so an identical retry must not be an error.
if existing == payload.addr:
return {"status": "ok"}
raise HTTPException(
status_code=400,
detail=(
f"Worker with dp_rank={payload.dp_rank}, "
f"tp_rank={payload.tp_rank}, pp_rank={payload.pp_rank} "
f"is already registered at "
f"{tp_entry[payload.pp_rank]}, "
f"{existing}, "
f"but still want to register at {payload.addr}"
),
)