[Bugfix][ROCm] Keep Mooncake bootstrap ports bound during startup (#58967)
Signed-off-by: Andreas Karatzas <andreas.karatzas@protonmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
6ebb5bd64f
commit
3acddf01e7
@@ -2329,6 +2329,10 @@ def _async_loop(loop: asyncio.AbstractEventLoop):
|
||||
|
||||
|
||||
def should_launch_bootstrap_server(vllm_config: VllmConfig) -> bool:
|
||||
if (kv_config := vllm_config.kv_transfer_config) is not None and (
|
||||
kv_config.get_from_extra_config("bootstrap_server_address", None) is not None
|
||||
):
|
||||
return False
|
||||
assert (parallel_config := vllm_config.parallel_config)
|
||||
# Only the TP=0, PP=0 worker of the designated engine should launch it.
|
||||
if get_tensor_model_parallel_rank() != 0:
|
||||
@@ -2351,6 +2355,17 @@ def get_mooncake_bootstrap_addr(vllm_config: VllmConfig) -> tuple[str, int]:
|
||||
This is only used by prefillers to register workers.
|
||||
Decoders should get addr from kv_transfer_params.
|
||||
"""
|
||||
if (kv_config := vllm_config.kv_transfer_config) is not None:
|
||||
address = kv_config.get_from_extra_config("bootstrap_server_address", None)
|
||||
if address is not None:
|
||||
from vllm.utils.network_utils import split_host_port
|
||||
|
||||
host, port = split_host_port(address)
|
||||
if not host or not 1 <= port <= 65535:
|
||||
raise ValueError(
|
||||
"bootstrap_server_address must specify a host and port"
|
||||
)
|
||||
return host, port
|
||||
assert (parallel_config := vllm_config.parallel_config)
|
||||
if parallel_config.local_engines_only:
|
||||
# In hybrid or external LB mode, connect to local server.
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
@@ -32,9 +33,7 @@ class EngineEntry:
|
||||
|
||||
|
||||
class MooncakeBootstrapServer:
|
||||
"""A centralized server running on the global rank 0 prefiller worker.
|
||||
Prefiller workers register their connection info (IP, port, ranks) here.
|
||||
"""
|
||||
"""A centralized registry for prefiller connection info (IP, port, ranks)."""
|
||||
|
||||
def __init__(self, host: str, port: int):
|
||||
self.workers: dict[int, EngineEntry] = {}
|
||||
@@ -45,6 +44,8 @@ class MooncakeBootstrapServer:
|
||||
self._register_routes()
|
||||
self.server_thread: threading.Thread | None = None
|
||||
self.server: uvicorn.Server | None = None
|
||||
self._socket: socket.socket | None = None
|
||||
self._startup_error: BaseException | None = None
|
||||
|
||||
def __del__(self):
|
||||
self.shutdown()
|
||||
@@ -54,27 +55,74 @@ class MooncakeBootstrapServer:
|
||||
self.app.post("/register")(self.register_worker)
|
||||
self.app.get("/query", response_model=dict[int, EngineEntry])(self.query)
|
||||
|
||||
def start(self):
|
||||
def start(self, *, timeout: float = 30.0):
|
||||
if self.server_thread:
|
||||
return
|
||||
if (
|
||||
self.server_thread.is_alive()
|
||||
and self.server is not None
|
||||
and self.server.started
|
||||
and not self.server.should_exit
|
||||
and self._socket is not None
|
||||
and self._socket.fileno() >= 0
|
||||
):
|
||||
return
|
||||
raise RuntimeError(
|
||||
"Mooncake bootstrap server is still starting or stopping"
|
||||
)
|
||||
|
||||
config = uvicorn.Config(app=self.app, host=self.host, port=self.port)
|
||||
self.server = uvicorn.Server(config=config)
|
||||
self.server_thread = threading.Thread(
|
||||
target=self.server.run, name="mooncake_bootstrap_server", daemon=True
|
||||
)
|
||||
self.server_thread.start()
|
||||
while not self.server.started:
|
||||
time.sleep(0.1) # Wait for the server to start
|
||||
try:
|
||||
# Bind in the caller so errors propagate, and retain the listener
|
||||
# when handing it to Uvicorn, including for automatically chosen ports.
|
||||
family = socket.AF_INET6 if ":" in self.host else socket.AF_INET
|
||||
self._socket = socket.create_server((self.host, self.port), family=family)
|
||||
self.port = self._socket.getsockname()[1]
|
||||
config = uvicorn.Config(app=self.app, host=self.host, port=self.port)
|
||||
server = self.server = uvicorn.Server(config=config)
|
||||
listener = self._socket
|
||||
self._startup_error = None
|
||||
|
||||
def run():
|
||||
try:
|
||||
server.run(sockets=[listener])
|
||||
except BaseException as exc:
|
||||
self._startup_error = exc
|
||||
|
||||
self.server_thread = threading.Thread(
|
||||
target=run, name="mooncake_bootstrap_server", daemon=True
|
||||
)
|
||||
self.server_thread.start()
|
||||
deadline = time.monotonic() + timeout
|
||||
while not server.started:
|
||||
if not self.server_thread.is_alive():
|
||||
raise RuntimeError(
|
||||
"Mooncake bootstrap server exited during startup"
|
||||
) from self._startup_error
|
||||
if time.monotonic() >= deadline:
|
||||
raise TimeoutError(
|
||||
"Mooncake bootstrap server did not start in time"
|
||||
)
|
||||
time.sleep(0.01)
|
||||
except BaseException:
|
||||
self.shutdown()
|
||||
raise
|
||||
logger.info("Mooncake Bootstrap Server started at %s:%d", self.host, self.port)
|
||||
|
||||
def shutdown(self):
|
||||
if self.server_thread is None or self.server is None or not self.server.started:
|
||||
return
|
||||
|
||||
self.server.should_exit = True
|
||||
self.server_thread.join()
|
||||
logger.info("Mooncake Bootstrap Server stopped.")
|
||||
was_started = self.server is not None and self.server.started
|
||||
if self.server is not None:
|
||||
self.server.should_exit = True
|
||||
if self.server_thread is not None and self.server_thread.ident is not None:
|
||||
self.server_thread.join(timeout=5)
|
||||
if self._socket is not None:
|
||||
self._socket.close()
|
||||
self._socket = None
|
||||
if self.server_thread is not None and self.server_thread.is_alive():
|
||||
logger.warning("Mooncake bootstrap server did not stop in time")
|
||||
else:
|
||||
self.server_thread = None
|
||||
self.server = None
|
||||
if was_started:
|
||||
logger.info("Mooncake Bootstrap Server stopped.")
|
||||
|
||||
async def register_worker(self, payload: RegisterWorkerPayload):
|
||||
"""Handles registration of a prefiller worker."""
|
||||
|
||||
Reference in New Issue
Block a user