[Bugfix]MooncakeConnect add query timeout and retry (#55923)
This commit is contained in:
@@ -49,12 +49,13 @@ cache), see [ECMooncakeConnector Usage Guide](mooncake_ec_connector_usage.md).
|
||||
- For headless instances, must be the same as the master instance
|
||||
- Each instance needs a unique port on its host; using the same port number across different hosts is fine
|
||||
|
||||
- `VLLM_MOONCAKE_CONNECTOR_TIMEOUT`: Per-request timeout (in seconds) for MooncakeConnector HTTP calls to the bootstrap server, including prefiller worker registration at startup. (Optional)
|
||||
- `VLLM_MOONCAKE_CONNECTOR_TIMEOUT`: Per-request timeout (in seconds) for MooncakeConnector HTTP calls to the bootstrap server, including prefiller worker registration at startup and metadata queries. (Optional)
|
||||
- Default: 30.0
|
||||
- Global rank 0 hosts the bootstrap server in the same process that mounts the Mooncake transfer engine's host segment, so a large `MOONCAKE_GLOBAL_SEGMENT_SIZE` can delay responses for several seconds during startup
|
||||
- Raise this if bootstrap server timeout warnings appear on hosts with very large host-memory segments
|
||||
- Bootstrap registration uses up to 3 attempts internally, with exponential backoff capped at 10 seconds; connection errors and timeouts are both retried
|
||||
- Once the attempts are exhausted the caller raises instead of blocking, so failures surface rather than hanging
|
||||
- Once registration attempts are exhausted the caller raises instead of blocking, so failures surface rather than hanging
|
||||
- Metadata queries reuse the same timeout and 3-attempt limit, with a fixed 100ms delay between attempts. If all attempts fail, the error is logged and waiting queries are notified
|
||||
|
||||
- `WITH_NVIDIA_PEERMEM`: Selects how mooncake registers GPU memory for RDMA. Read by mooncake, not vLLM.
|
||||
- Default: 1, which uses `ibv_reg_mr()` and requires the `nvidia-peermem` kernel module to be loaded
|
||||
|
||||
@@ -2438,9 +2438,30 @@ class MooncakeConnectorWorker:
|
||||
|
||||
async def _connect_to_prefiller_bootstrap(self, remote_bootstrap_addr: str):
|
||||
url = remote_bootstrap_addr + "/query"
|
||||
max_attempts = _BOOTSTRAP_MAX_ATTEMPTS
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.get(url)
|
||||
async with httpx.AsyncClient(
|
||||
timeout=envs.VLLM_MOONCAKE_CONNECTOR_TIMEOUT
|
||||
) as client:
|
||||
retry_delay = 0.1
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
try:
|
||||
response = await client.get(url)
|
||||
break
|
||||
except httpx.RequestError as e:
|
||||
if attempt == max_attempts:
|
||||
raise
|
||||
logger.warning(
|
||||
"Bootstrap query to %s failed on attempt %d/%d "
|
||||
"(%s: %s); retrying in %.1f seconds",
|
||||
remote_bootstrap_addr,
|
||||
attempt,
|
||||
max_attempts,
|
||||
type(e).__name__,
|
||||
e,
|
||||
retry_delay,
|
||||
)
|
||||
await asyncio.sleep(retry_delay)
|
||||
response.raise_for_status()
|
||||
data: dict = response.json()
|
||||
for _, dp_entry in data.items():
|
||||
@@ -2455,8 +2476,9 @@ class MooncakeConnectorWorker:
|
||||
self._tp_size[remote_engine_id] = len(dp_entry["worker_addr"])
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Failed to connect to bootstrap server %s: %s",
|
||||
"Failed to connect to bootstrap server %s (%s): %s",
|
||||
remote_bootstrap_addr,
|
||||
type(e).__name__,
|
||||
e,
|
||||
)
|
||||
|
||||
|
||||
@@ -1305,7 +1305,7 @@ def _engram_select_rows(
|
||||
) -> None:
|
||||
"""Copy one token window out of a rank-major gathered buffer.
|
||||
|
||||
Both gathers land rank-major ([rank][token][local width]); this walks the
|
||||
Both gathers land rank-major (`[rank][token][local width]`); this walks the
|
||||
window the rank keeps and lays its ranks out side by side as width.
|
||||
"""
|
||||
if output.numel() == 0:
|
||||
|
||||
Reference in New Issue
Block a user