From bc713cf8cc861c0d0ccfe46c9ce322ab81993ace Mon Sep 17 00:00:00 2001 From: Chao Lei Date: Fri, 9 Oct 2026 17:19:33 +0800 Subject: [PATCH] [Bugfix]MooncakeConnect add query timeout and retry (#55923) --- docs/features/mooncake_connector_usage.md | 5 ++-- .../v1/mooncake/mooncake_connector.py | 28 +++++++++++++++++-- vllm/models/deepseek_v41/common/engram.py | 2 +- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/docs/features/mooncake_connector_usage.md b/docs/features/mooncake_connector_usage.md index 866d6d6bd4..00bfdeabc1 100644 --- a/docs/features/mooncake_connector_usage.md +++ b/docs/features/mooncake_connector_usage.md @@ -49,12 +49,13 @@ cache), see [ECMooncakeConnector Usage Guide](mooncake_ec_connector_usage.md). - For headless instances, must be the same as the master instance - Each instance needs a unique port on its host; using the same port number across different hosts is fine -- `VLLM_MOONCAKE_CONNECTOR_TIMEOUT`: Per-request timeout (in seconds) for MooncakeConnector HTTP calls to the bootstrap server, including prefiller worker registration at startup. (Optional) +- `VLLM_MOONCAKE_CONNECTOR_TIMEOUT`: Per-request timeout (in seconds) for MooncakeConnector HTTP calls to the bootstrap server, including prefiller worker registration at startup and metadata queries. (Optional) - Default: 30.0 - Global rank 0 hosts the bootstrap server in the same process that mounts the Mooncake transfer engine's host segment, so a large `MOONCAKE_GLOBAL_SEGMENT_SIZE` can delay responses for several seconds during startup - Raise this if bootstrap server timeout warnings appear on hosts with very large host-memory segments - Bootstrap registration uses up to 3 attempts internally, with exponential backoff capped at 10 seconds; connection errors and timeouts are both retried - - Once the attempts are exhausted the caller raises instead of blocking, so failures surface rather than hanging + - Once registration attempts are exhausted the caller raises instead of blocking, so failures surface rather than hanging + - Metadata queries reuse the same timeout and 3-attempt limit, with a fixed 100ms delay between attempts. If all attempts fail, the error is logged and waiting queries are notified - `WITH_NVIDIA_PEERMEM`: Selects how mooncake registers GPU memory for RDMA. Read by mooncake, not vLLM. - Default: 1, which uses `ibv_reg_mr()` and requires the `nvidia-peermem` kernel module to be loaded diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py index 8300cd950b..9d5194f33e 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py @@ -2438,9 +2438,30 @@ class MooncakeConnectorWorker: async def _connect_to_prefiller_bootstrap(self, remote_bootstrap_addr: str): url = remote_bootstrap_addr + "/query" + max_attempts = _BOOTSTRAP_MAX_ATTEMPTS try: - async with httpx.AsyncClient() as client: - response = await client.get(url) + async with httpx.AsyncClient( + timeout=envs.VLLM_MOONCAKE_CONNECTOR_TIMEOUT + ) as client: + retry_delay = 0.1 + for attempt in range(1, max_attempts + 1): + try: + response = await client.get(url) + break + except httpx.RequestError as e: + if attempt == max_attempts: + raise + logger.warning( + "Bootstrap query to %s failed on attempt %d/%d " + "(%s: %s); retrying in %.1f seconds", + remote_bootstrap_addr, + attempt, + max_attempts, + type(e).__name__, + e, + retry_delay, + ) + await asyncio.sleep(retry_delay) response.raise_for_status() data: dict = response.json() for _, dp_entry in data.items(): @@ -2455,8 +2476,9 @@ class MooncakeConnectorWorker: self._tp_size[remote_engine_id] = len(dp_entry["worker_addr"]) except Exception as e: logger.error( - "Failed to connect to bootstrap server %s: %s", + "Failed to connect to bootstrap server %s (%s): %s", remote_bootstrap_addr, + type(e).__name__, e, ) diff --git a/vllm/models/deepseek_v41/common/engram.py b/vllm/models/deepseek_v41/common/engram.py index 0e160c8b06..236235dfa8 100644 --- a/vllm/models/deepseek_v41/common/engram.py +++ b/vllm/models/deepseek_v41/common/engram.py @@ -1305,7 +1305,7 @@ def _engram_select_rows( ) -> None: """Copy one token window out of a rank-major gathered buffer. - Both gathers land rank-major ([rank][token][local width]); this walks the + Both gathers land rank-major (`[rank][token][local width]`); this walks the window the rank keeps and lays its ranks out side by side as width. """ if output.numel() == 0: