[Bugfix][ROCm] Keep Mooncake bootstrap ports bound during startup (#58967)

Signed-off-by: Andreas Karatzas <andreas.karatzas@protonmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Andreas Karatzas
2026-09-29 04:14:26 -05:00
committed by GitHub
co-authored by OpenAI Codex
parent 6ebb5bd64f
commit 3acddf01e7
5 changed files with 226 additions and 36 deletions
@@ -2329,6 +2329,10 @@ def _async_loop(loop: asyncio.AbstractEventLoop):
def should_launch_bootstrap_server(vllm_config: VllmConfig) -> bool:
if (kv_config := vllm_config.kv_transfer_config) is not None and (
kv_config.get_from_extra_config("bootstrap_server_address", None) is not None
):
return False
assert (parallel_config := vllm_config.parallel_config)
# Only the TP=0, PP=0 worker of the designated engine should launch it.
if get_tensor_model_parallel_rank() != 0:
@@ -2351,6 +2355,17 @@ def get_mooncake_bootstrap_addr(vllm_config: VllmConfig) -> tuple[str, int]:
This is only used by prefillers to register workers.
Decoders should get addr from kv_transfer_params.
"""
if (kv_config := vllm_config.kv_transfer_config) is not None:
address = kv_config.get_from_extra_config("bootstrap_server_address", None)
if address is not None:
from vllm.utils.network_utils import split_host_port
host, port = split_host_port(address)
if not host or not 1 <= port <= 65535:
raise ValueError(
"bootstrap_server_address must specify a host and port"
)
return host, port
assert (parallel_config := vllm_config.parallel_config)
if parallel_config.local_engines_only:
# In hybrid or external LB mode, connect to local server.
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import socket
import threading
import time
from dataclasses import dataclass
@@ -32,9 +33,7 @@ class EngineEntry:
class MooncakeBootstrapServer:
"""A centralized server running on the global rank 0 prefiller worker.
Prefiller workers register their connection info (IP, port, ranks) here.
"""
"""A centralized registry for prefiller connection info (IP, port, ranks)."""
def __init__(self, host: str, port: int):
self.workers: dict[int, EngineEntry] = {}
@@ -45,6 +44,8 @@ class MooncakeBootstrapServer:
self._register_routes()
self.server_thread: threading.Thread | None = None
self.server: uvicorn.Server | None = None
self._socket: socket.socket | None = None
self._startup_error: BaseException | None = None
def __del__(self):
self.shutdown()
@@ -54,27 +55,74 @@ class MooncakeBootstrapServer:
self.app.post("/register")(self.register_worker)
self.app.get("/query", response_model=dict[int, EngineEntry])(self.query)
def start(self):
def start(self, *, timeout: float = 30.0):
if self.server_thread:
return
if (
self.server_thread.is_alive()
and self.server is not None
and self.server.started
and not self.server.should_exit
and self._socket is not None
and self._socket.fileno() >= 0
):
return
raise RuntimeError(
"Mooncake bootstrap server is still starting or stopping"
)
config = uvicorn.Config(app=self.app, host=self.host, port=self.port)
self.server = uvicorn.Server(config=config)
self.server_thread = threading.Thread(
target=self.server.run, name="mooncake_bootstrap_server", daemon=True
)
self.server_thread.start()
while not self.server.started:
time.sleep(0.1) # Wait for the server to start
try:
# Bind in the caller so errors propagate, and retain the listener
# when handing it to Uvicorn, including for automatically chosen ports.
family = socket.AF_INET6 if ":" in self.host else socket.AF_INET
self._socket = socket.create_server((self.host, self.port), family=family)
self.port = self._socket.getsockname()[1]
config = uvicorn.Config(app=self.app, host=self.host, port=self.port)
server = self.server = uvicorn.Server(config=config)
listener = self._socket
self._startup_error = None
def run():
try:
server.run(sockets=[listener])
except BaseException as exc:
self._startup_error = exc
self.server_thread = threading.Thread(
target=run, name="mooncake_bootstrap_server", daemon=True
)
self.server_thread.start()
deadline = time.monotonic() + timeout
while not server.started:
if not self.server_thread.is_alive():
raise RuntimeError(
"Mooncake bootstrap server exited during startup"
) from self._startup_error
if time.monotonic() >= deadline:
raise TimeoutError(
"Mooncake bootstrap server did not start in time"
)
time.sleep(0.01)
except BaseException:
self.shutdown()
raise
logger.info("Mooncake Bootstrap Server started at %s:%d", self.host, self.port)
def shutdown(self):
if self.server_thread is None or self.server is None or not self.server.started:
return
self.server.should_exit = True
self.server_thread.join()
logger.info("Mooncake Bootstrap Server stopped.")
was_started = self.server is not None and self.server.started
if self.server is not None:
self.server.should_exit = True
if self.server_thread is not None and self.server_thread.ident is not None:
self.server_thread.join(timeout=5)
if self._socket is not None:
self._socket.close()
self._socket = None
if self.server_thread is not None and self.server_thread.is_alive():
logger.warning("Mooncake bootstrap server did not stop in time")
else:
self.server_thread = None
self.server = None
if was_started:
logger.info("Mooncake Bootstrap Server stopped.")
async def register_worker(self, payload: RegisterWorkerPayload):
"""Handles registration of a prefiller worker."""