This commit is contained in:
@@ -10,7 +10,7 @@ from collections.abc import Collection
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from enum import IntEnum
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import msgspec
|
||||
import numpy as np
|
||||
@@ -70,6 +70,8 @@ from vllm.v1.worker.utils import select_common_block_size
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_BOOTSTRAP_MAX_ATTEMPTS: Final[int] = 3
|
||||
|
||||
try:
|
||||
from mooncake.engine import TransferEngine
|
||||
except ImportError:
|
||||
@@ -1147,24 +1149,44 @@ class MooncakeConnectorWorker:
|
||||
pp_rank=self.pp_rank,
|
||||
addr=worker_addr,
|
||||
)
|
||||
while True:
|
||||
timeout = envs.VLLM_MOONCAKE_CONNECTOR_TIMEOUT
|
||||
max_attempts = _BOOTSTRAP_MAX_ATTEMPTS
|
||||
backoff = 1.0
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
response = await client.post(url, json=payload.model_dump())
|
||||
response.raise_for_status()
|
||||
logger.debug("Successfully registered with bootstrap server at %s", url)
|
||||
break
|
||||
except httpx.ConnectError:
|
||||
# Bootstrap server not ready, wait for a while and retry.
|
||||
await asyncio.sleep(1)
|
||||
return
|
||||
except (httpx.ConnectError, httpx.TimeoutException) as e:
|
||||
# Rank 0 may be busy registering a large memory segment.
|
||||
if attempt == max_attempts:
|
||||
break
|
||||
logger.warning(
|
||||
"Bootstrap registration attempt %d/%d for %s failed with %s: %s. "
|
||||
"Retrying in %.1fs.",
|
||||
attempt,
|
||||
max_attempts,
|
||||
payload,
|
||||
type(e).__name__,
|
||||
e,
|
||||
backoff,
|
||||
)
|
||||
await asyncio.sleep(backoff)
|
||||
backoff = min(backoff * 2, 10.0)
|
||||
except Exception as e:
|
||||
err_msg = (
|
||||
e.response.text if isinstance(e, httpx.HTTPStatusError) else str(e)
|
||||
)
|
||||
logger.error(
|
||||
logger.exception(
|
||||
"Error registering %s with bootstrap server: %s", payload, err_msg
|
||||
)
|
||||
raise e
|
||||
raise
|
||||
raise RuntimeError(
|
||||
f"Failed to register {payload} with the Mooncake bootstrap server at "
|
||||
f"{url} after {max_attempts} attempts."
|
||||
)
|
||||
|
||||
async def _mooncake_sender_listener(self, ready_event: threading.Event):
|
||||
"""Background thread that listens for Mooncake requests, dispatches them
|
||||
@@ -1800,11 +1822,29 @@ class MooncakeConnectorWorker:
|
||||
if self.is_kv_consumer:
|
||||
return
|
||||
|
||||
# httpx applies the timeout to each phase (connect, read, write, pool)
|
||||
# rather than to the whole request, so budget for all four.
|
||||
register_timeout = envs.VLLM_MOONCAKE_CONNECTOR_TIMEOUT
|
||||
max_attempts = _BOOTSTRAP_MAX_ATTEMPTS
|
||||
backoff_total = sum(min(2.0**i, 10.0) for i in range(max_attempts - 1))
|
||||
ready_timeout = max_attempts * 4 * register_timeout + backoff_total + 30.0
|
||||
ready_event = threading.Event()
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
fut = asyncio.run_coroutine_threadsafe(
|
||||
self._mooncake_sender_listener(ready_event), self.sender_loop
|
||||
)
|
||||
ready_event.wait() # Wait for listener ZMQ socket to be ready.
|
||||
deadline = time.monotonic() + ready_timeout
|
||||
while not ready_event.wait(timeout=1.0):
|
||||
if fut.done():
|
||||
fut.result()
|
||||
raise RuntimeError(
|
||||
"Mooncake sender listener exited before becoming ready."
|
||||
)
|
||||
if time.monotonic() > deadline:
|
||||
fut.cancel()
|
||||
raise RuntimeError(
|
||||
"Mooncake sender listener did not become ready within "
|
||||
f"{ready_timeout:.0f}s."
|
||||
)
|
||||
|
||||
async def fetch_finished_recving_reqs(self) -> set[ReqId]:
|
||||
finished_recving_reqs = self.finished_recving_reqs
|
||||
|
||||
@@ -97,14 +97,19 @@ class MooncakeBootstrapServer:
|
||||
dp_entry.worker_addr[payload.tp_rank] = {}
|
||||
|
||||
tp_entry = dp_entry.worker_addr[payload.tp_rank]
|
||||
if payload.pp_rank in tp_entry:
|
||||
existing = tp_entry.get(payload.pp_rank)
|
||||
if existing is not None:
|
||||
# A client timeout can fire after the server recorded the
|
||||
# registration so an identical retry must not be an error.
|
||||
if existing == payload.addr:
|
||||
return {"status": "ok"}
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
f"Worker with dp_rank={payload.dp_rank}, "
|
||||
f"tp_rank={payload.tp_rank}, pp_rank={payload.pp_rank} "
|
||||
f"is already registered at "
|
||||
f"{tp_entry[payload.pp_rank]}, "
|
||||
f"{existing}, "
|
||||
f"but still want to register at {payload.addr}"
|
||||
),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user