[Bugfix][ROCm] Keep Mooncake bootstrap ports bound during startup (#58967)

Signed-off-by: Andreas Karatzas <andreas.karatzas@protonmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Andreas Karatzas
2026-09-29 04:14:26 -05:00
committed by GitHub
co-authored by OpenAI Codex
parent 6ebb5bd64f
commit 3acddf01e7
5 changed files with 226 additions and 36 deletions
@@ -76,6 +76,7 @@ Now you can send requests to the proxy server through port 8000.
- **num_workers**: Size of thread pool for one prefiller worker to transfer KV caches by mooncake. (default 10)
- **mooncake_protocol**: Mooncake connector protocol. (default "rdma")
- **device_name**: Comma-separated whitelist of RDMA devices (e.g. `"mlx5_0,mlx5_1"`) to restrict topology discovery to. Empty discovers every device. Useful on hosts exposing a mix of InfiniBand and RoCE ports, where both peers must settle on the same link layer.
- **bootstrap_server_address**: Optional `"host:port"` of an already running Mooncake bootstrap server. All prefiller ranks register with this endpoint instead of launching an embedded server. The caller owns the server lifetime and must configure the proxy to use the same endpoint. The example proxy assumes the bootstrap server is on the prefiller API host; a different host requires a proxy that supplies that address in `remote_bootstrap_addr`. Omit this option to use the embedded server and `VLLM_MOONCAKE_BOOTSTRAP_PORT`.
## Example Scripts/Code
@@ -158,8 +158,18 @@ def run_rocm_pd_accuracy(connector, model, prefill_tp, decode_tp, monkeypatch):
"VLLM_ROCM_USE_AITER": "1",
"VLLM_KV_CACHE_LAYOUT": "HND",
}
bootstrap_port = get_open_port()
with contextlib.ExitStack() as stack:
bootstrap_port = 0
if connector == "mooncake":
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import ( # noqa: E501
MooncakeBootstrapServer,
)
bootstrap = MooncakeBootstrapServer("127.0.0.1", 0)
stack.callback(bootstrap.shutdown)
bootstrap.start()
bootstrap_port = bootstrap.port
extra["bootstrap_server_address"] = f"127.0.0.1:{bootstrap_port}"
servers: list[RemoteOpenAIServer] = []
stack.callback(RemoteOpenAIServer.shutdown_many, servers)
for role, tp, assigned in (
@@ -176,7 +186,6 @@ def run_rocm_pd_accuracy(connector, model, prefill_tp, decode_tp, monkeypatch):
selector: ",".join(assigned),
"VLLM_PORT": str(get_open_port()),
"VLLM_NIXL_SIDE_CHANNEL_PORT": str(get_open_port()),
"VLLM_MOONCAKE_BOOTSTRAP_PORT": str(bootstrap_port),
}
servers.append(
RemoteOpenAIServer(
@@ -3,6 +3,8 @@
import asyncio
import contextlib
import errno
import socket
import threading
import time
from types import SimpleNamespace
@@ -15,8 +17,11 @@ import zmq.asyncio
from tests.v1.attention.utils import dense_kv_cache_views
from vllm import envs
from vllm.config import set_current_vllm_config
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import mooncake_connector
from vllm.config import KVTransferConfig, set_current_vllm_config
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import (
mooncake_connector,
mooncake_utils,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import (
KVConnectorRole,
MooncakeConnector,
@@ -38,7 +43,6 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import
MooncakeBootstrapServer,
RegisterWorkerPayload,
)
from vllm.utils.network_utils import get_open_port
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
@@ -535,15 +539,99 @@ def test_prompt_less_than_block_size():
@pytest.fixture
def bootstrap_server():
"""Fixture to launch and cleanup a Mooncake Bootstrap HTTP Server."""
port = get_open_port()
server = MooncakeBootstrapServer("127.0.0.1", port)
server.start()
yield server
server.shutdown()
server = MooncakeBootstrapServer("127.0.0.1", 0)
try:
server.start()
yield server
finally:
server.shutdown()
def test_bootstrap_server_occupied_port_raises_to_caller():
with socket.create_server(("127.0.0.1", 0)) as listener:
server = MooncakeBootstrapServer("127.0.0.1", listener.getsockname()[1])
try:
with pytest.raises(OSError) as exc_info:
server.start(timeout=0.05)
assert exc_info.value.errno == errno.EADDRINUSE
finally:
server.shutdown()
def test_bootstrap_server_retains_and_releases_assigned_port(bootstrap_server):
port = bootstrap_server.port
assert port > 0
with (
pytest.raises(OSError) as exc_info,
socket.create_server(("127.0.0.1", port)),
):
pass
assert exc_info.value.errno == errno.EADDRINUSE
bootstrap_server.shutdown()
with socket.create_server(("127.0.0.1", port)):
pass
@pytest.mark.parametrize("failure", ["exit", "error", "timeout"])
def test_bootstrap_server_startup_failure_releases_port(monkeypatch, failure):
stop = threading.Event()
def run_without_starting(server, *, sockets):
if failure == "error":
raise SystemExit(1)
if failure == "timeout":
while not server.should_exit and not stop.wait(0.01):
pass
monkeypatch.setattr(mooncake_utils.uvicorn.Server, "run", run_without_starting)
server = MooncakeBootstrapServer("127.0.0.1", 0)
try:
error = TimeoutError if failure == "timeout" else RuntimeError
with pytest.raises(error):
server.start(timeout=0.05 if failure == "timeout" else 1.0)
assert server.port > 0
with socket.create_server(("127.0.0.1", server.port)):
pass
finally:
stop.set()
server.shutdown()
@pytest.mark.parametrize("state", ["stopping", "exited"])
def test_bootstrap_server_rejects_stale_started_state(monkeypatch, state):
release = threading.Event()
def run_until_released(server, *, sockets):
server.started = True
release.wait()
monkeypatch.setattr(mooncake_utils.uvicorn.Server, "run", run_until_released)
server = MooncakeBootstrapServer("127.0.0.1", 0)
try:
server.start()
thread = server.server_thread
assert thread is not None
if state == "stopping":
with patch.object(thread, "join", return_value=None):
server.shutdown()
else:
release.set()
thread.join(timeout=5)
assert not thread.is_alive()
with pytest.raises(RuntimeError, match="still starting or stopping"):
server.start()
finally:
release.set()
server.shutdown()
@pytest.mark.asyncio
async def test_register_worker_recovers_from_slow_bootstrap_server(monkeypatch):
@pytest.mark.parametrize("external", [False, True])
async def test_register_worker_recovers_from_slow_bootstrap_server(
monkeypatch, external
):
"""End-to-end: a real HTTP server that responds slower than the configured
timeout must be retried on the wire, not treated as fatal."""
original = MooncakeBootstrapServer.register_worker
@@ -562,20 +650,24 @@ async def test_register_worker_recovers_from_slow_bootstrap_server(monkeypatch):
monkeypatch.setattr(MooncakeBootstrapServer, "register_worker", slow_register)
port = get_open_port()
monkeypatch.setenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", str(port))
monkeypatch.setenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "0.5")
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 5)
server = MooncakeBootstrapServer("127.0.0.1", port)
server = MooncakeBootstrapServer("127.0.0.1", 0)
server.start()
try:
monkeypatch.setenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", str(server.port))
worker = _make_local_register_worker_stub()
if external:
worker.vllm_config = _make_bootstrap_vllm_config(
bootstrap_server_address=f"127.0.0.1:{server.port}"
)
assert not should_launch_bootstrap_server(worker.vllm_config)
await MooncakeConnectorWorker.register_worker_with_bootstrap(worker)
assert state["calls"] == 2
async with httpx.AsyncClient() as client:
response = await client.get(f"http://127.0.0.1:{port}/query")
response = await client.get(f"http://127.0.0.1:{server.port}/query")
assert response.status_code == 200
assert response.json()["0"]["engine_id"] == "eng-1"
finally:
@@ -666,8 +758,16 @@ def _make_bootstrap_vllm_config(
data_parallel_rank_local: int = 0,
data_parallel_index: int = 0,
nnodes_within_dp: int = 1,
bootstrap_server_address: str | None = None,
) -> SimpleNamespace:
return SimpleNamespace(
kv_transfer_config=KVTransferConfig(
kv_connector="MooncakeConnector",
kv_role="kv_producer",
kv_connector_extra_config={
"bootstrap_server_address": bootstrap_server_address
},
),
parallel_config=SimpleNamespace(
local_engines_only=local_engines_only,
data_parallel_rank_local=data_parallel_rank_local,
@@ -675,13 +775,14 @@ def _make_bootstrap_vllm_config(
nnodes_within_dp=nnodes_within_dp,
master_addr="model-parallel-master",
data_parallel_master_ip="data-parallel-master",
)
),
)
def _make_local_register_worker_stub():
return SimpleNamespace(
vllm_config=SimpleNamespace(
kv_transfer_config=None,
parallel_config=SimpleNamespace(
local_engines_only=False,
data_parallel_rank_local=0,
@@ -689,7 +790,7 @@ def _make_local_register_worker_stub():
nnodes_within_dp=1,
master_addr="127.0.0.1",
data_parallel_master_ip="127.0.0.1",
)
),
),
hostname="127.0.0.1",
side_channel_port=1234,
@@ -889,6 +990,22 @@ def test_get_mooncake_bootstrap_addr_selects_expected_host(
)
@pytest.mark.parametrize(
("address", "expected"),
[("external:12345", ("external", 12345)), ("[::1]:12345", ("::1", 12345))],
)
def test_get_mooncake_bootstrap_addr_uses_external_endpoint(address, expected):
config = _make_bootstrap_vllm_config(bootstrap_server_address=address)
assert get_mooncake_bootstrap_addr(config) == expected
@pytest.mark.parametrize("address", [":12345", "host:0", "host:65536"])
def test_get_mooncake_bootstrap_addr_rejects_invalid_external_address(address):
config = _make_bootstrap_vllm_config(bootstrap_server_address=address)
with pytest.raises(ValueError, match="bootstrap_server_address"):
get_mooncake_bootstrap_addr(config)
def test_scheduler_request_finished():
"""Tests the scheduler-side logic when a request finishes.
@@ -2329,6 +2329,10 @@ def _async_loop(loop: asyncio.AbstractEventLoop):
def should_launch_bootstrap_server(vllm_config: VllmConfig) -> bool:
if (kv_config := vllm_config.kv_transfer_config) is not None and (
kv_config.get_from_extra_config("bootstrap_server_address", None) is not None
):
return False
assert (parallel_config := vllm_config.parallel_config)
# Only the TP=0, PP=0 worker of the designated engine should launch it.
if get_tensor_model_parallel_rank() != 0:
@@ -2351,6 +2355,17 @@ def get_mooncake_bootstrap_addr(vllm_config: VllmConfig) -> tuple[str, int]:
This is only used by prefillers to register workers.
Decoders should get addr from kv_transfer_params.
"""
if (kv_config := vllm_config.kv_transfer_config) is not None:
address = kv_config.get_from_extra_config("bootstrap_server_address", None)
if address is not None:
from vllm.utils.network_utils import split_host_port
host, port = split_host_port(address)
if not host or not 1 <= port <= 65535:
raise ValueError(
"bootstrap_server_address must specify a host and port"
)
return host, port
assert (parallel_config := vllm_config.parallel_config)
if parallel_config.local_engines_only:
# In hybrid or external LB mode, connect to local server.
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import socket
import threading
import time
from dataclasses import dataclass
@@ -32,9 +33,7 @@ class EngineEntry:
class MooncakeBootstrapServer:
"""A centralized server running on the global rank 0 prefiller worker.
Prefiller workers register their connection info (IP, port, ranks) here.
"""
"""A centralized registry for prefiller connection info (IP, port, ranks)."""
def __init__(self, host: str, port: int):
self.workers: dict[int, EngineEntry] = {}
@@ -45,6 +44,8 @@ class MooncakeBootstrapServer:
self._register_routes()
self.server_thread: threading.Thread | None = None
self.server: uvicorn.Server | None = None
self._socket: socket.socket | None = None
self._startup_error: BaseException | None = None
def __del__(self):
self.shutdown()
@@ -54,27 +55,74 @@ class MooncakeBootstrapServer:
self.app.post("/register")(self.register_worker)
self.app.get("/query", response_model=dict[int, EngineEntry])(self.query)
def start(self):
def start(self, *, timeout: float = 30.0):
if self.server_thread:
return
if (
self.server_thread.is_alive()
and self.server is not None
and self.server.started
and not self.server.should_exit
and self._socket is not None
and self._socket.fileno() >= 0
):
return
raise RuntimeError(
"Mooncake bootstrap server is still starting or stopping"
)
config = uvicorn.Config(app=self.app, host=self.host, port=self.port)
self.server = uvicorn.Server(config=config)
self.server_thread = threading.Thread(
target=self.server.run, name="mooncake_bootstrap_server", daemon=True
)
self.server_thread.start()
while not self.server.started:
time.sleep(0.1) # Wait for the server to start
try:
# Bind in the caller so errors propagate, and retain the listener
# when handing it to Uvicorn, including for automatically chosen ports.
family = socket.AF_INET6 if ":" in self.host else socket.AF_INET
self._socket = socket.create_server((self.host, self.port), family=family)
self.port = self._socket.getsockname()[1]
config = uvicorn.Config(app=self.app, host=self.host, port=self.port)
server = self.server = uvicorn.Server(config=config)
listener = self._socket
self._startup_error = None
def run():
try:
server.run(sockets=[listener])
except BaseException as exc:
self._startup_error = exc
self.server_thread = threading.Thread(
target=run, name="mooncake_bootstrap_server", daemon=True
)
self.server_thread.start()
deadline = time.monotonic() + timeout
while not server.started:
if not self.server_thread.is_alive():
raise RuntimeError(
"Mooncake bootstrap server exited during startup"
) from self._startup_error
if time.monotonic() >= deadline:
raise TimeoutError(
"Mooncake bootstrap server did not start in time"
)
time.sleep(0.01)
except BaseException:
self.shutdown()
raise
logger.info("Mooncake Bootstrap Server started at %s:%d", self.host, self.port)
def shutdown(self):
if self.server_thread is None or self.server is None or not self.server.started:
return
self.server.should_exit = True
self.server_thread.join()
logger.info("Mooncake Bootstrap Server stopped.")
was_started = self.server is not None and self.server.started
if self.server is not None:
self.server.should_exit = True
if self.server_thread is not None and self.server_thread.ident is not None:
self.server_thread.join(timeout=5)
if self._socket is not None:
self._socket.close()
self._socket = None
if self.server_thread is not None and self.server_thread.is_alive():
logger.warning("Mooncake bootstrap server did not stop in time")
else:
self.server_thread = None
self.server = None
if was_started:
logger.info("Mooncake Bootstrap Server stopped.")
async def register_worker(self, payload: RegisterWorkerPayload):
"""Handles registration of a prefiller worker."""