From 3acddf01e709b13693d3a90d25fdb74d8124049c Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Tue, 29 Sep 2026 04:14:26 -0500 Subject: [PATCH] [Bugfix][ROCm] Keep Mooncake bootstrap ports bound during startup (#58967) Signed-off-by: Andreas Karatzas Co-authored-by: OpenAI Codex --- docs/features/mooncake_connector_usage.md | 1 + .../v1/kv_connector/rocm_pd_accuracy_utils.py | 13 +- .../unit/test_mooncake_connector.py | 147 ++++++++++++++++-- .../v1/mooncake/mooncake_connector.py | 15 ++ .../v1/mooncake/mooncake_utils.py | 86 +++++++--- 5 files changed, 226 insertions(+), 36 deletions(-) diff --git a/docs/features/mooncake_connector_usage.md b/docs/features/mooncake_connector_usage.md index 7c5852a68b..701a1d0049 100644 --- a/docs/features/mooncake_connector_usage.md +++ b/docs/features/mooncake_connector_usage.md @@ -76,6 +76,7 @@ Now you can send requests to the proxy server through port 8000. - **num_workers**: Size of thread pool for one prefiller worker to transfer KV caches by mooncake. (default 10) - **mooncake_protocol**: Mooncake connector protocol. (default "rdma") - **device_name**: Comma-separated whitelist of RDMA devices (e.g. `"mlx5_0,mlx5_1"`) to restrict topology discovery to. Empty discovers every device. Useful on hosts exposing a mix of InfiniBand and RoCE ports, where both peers must settle on the same link layer. +- **bootstrap_server_address**: Optional `"host:port"` of an already running Mooncake bootstrap server. All prefiller ranks register with this endpoint instead of launching an embedded server. The caller owns the server lifetime and must configure the proxy to use the same endpoint. The example proxy assumes the bootstrap server is on the prefiller API host; a different host requires a proxy that supplies that address in `remote_bootstrap_addr`. Omit this option to use the embedded server and `VLLM_MOONCAKE_BOOTSTRAP_PORT`. ## Example Scripts/Code diff --git a/tests/v1/kv_connector/rocm_pd_accuracy_utils.py b/tests/v1/kv_connector/rocm_pd_accuracy_utils.py index e515c0a0bd..37d56ef6e4 100644 --- a/tests/v1/kv_connector/rocm_pd_accuracy_utils.py +++ b/tests/v1/kv_connector/rocm_pd_accuracy_utils.py @@ -158,8 +158,18 @@ def run_rocm_pd_accuracy(connector, model, prefill_tp, decode_tp, monkeypatch): "VLLM_ROCM_USE_AITER": "1", "VLLM_KV_CACHE_LAYOUT": "HND", } - bootstrap_port = get_open_port() with contextlib.ExitStack() as stack: + bootstrap_port = 0 + if connector == "mooncake": + from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import ( # noqa: E501 + MooncakeBootstrapServer, + ) + + bootstrap = MooncakeBootstrapServer("127.0.0.1", 0) + stack.callback(bootstrap.shutdown) + bootstrap.start() + bootstrap_port = bootstrap.port + extra["bootstrap_server_address"] = f"127.0.0.1:{bootstrap_port}" servers: list[RemoteOpenAIServer] = [] stack.callback(RemoteOpenAIServer.shutdown_many, servers) for role, tp, assigned in ( @@ -176,7 +186,6 @@ def run_rocm_pd_accuracy(connector, model, prefill_tp, decode_tp, monkeypatch): selector: ",".join(assigned), "VLLM_PORT": str(get_open_port()), "VLLM_NIXL_SIDE_CHANNEL_PORT": str(get_open_port()), - "VLLM_MOONCAKE_BOOTSTRAP_PORT": str(bootstrap_port), } servers.append( RemoteOpenAIServer( diff --git a/tests/v1/kv_connector/unit/test_mooncake_connector.py b/tests/v1/kv_connector/unit/test_mooncake_connector.py index 5a5fffac3d..be70a46bf7 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_connector.py +++ b/tests/v1/kv_connector/unit/test_mooncake_connector.py @@ -3,6 +3,8 @@ import asyncio import contextlib +import errno +import socket import threading import time from types import SimpleNamespace @@ -15,8 +17,11 @@ import zmq.asyncio from tests.v1.attention.utils import dense_kv_cache_views from vllm import envs -from vllm.config import set_current_vllm_config -from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import mooncake_connector +from vllm.config import KVTransferConfig, set_current_vllm_config +from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import ( + mooncake_connector, + mooncake_utils, +) from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import ( KVConnectorRole, MooncakeConnector, @@ -38,7 +43,6 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import MooncakeBootstrapServer, RegisterWorkerPayload, ) -from vllm.utils.network_utils import get_open_port from vllm.v1.kv_cache_interface import ( FullAttentionSpec, KVCacheConfig, @@ -535,15 +539,99 @@ def test_prompt_less_than_block_size(): @pytest.fixture def bootstrap_server(): """Fixture to launch and cleanup a Mooncake Bootstrap HTTP Server.""" - port = get_open_port() - server = MooncakeBootstrapServer("127.0.0.1", port) - server.start() - yield server - server.shutdown() + server = MooncakeBootstrapServer("127.0.0.1", 0) + try: + server.start() + yield server + finally: + server.shutdown() + + +def test_bootstrap_server_occupied_port_raises_to_caller(): + with socket.create_server(("127.0.0.1", 0)) as listener: + server = MooncakeBootstrapServer("127.0.0.1", listener.getsockname()[1]) + try: + with pytest.raises(OSError) as exc_info: + server.start(timeout=0.05) + assert exc_info.value.errno == errno.EADDRINUSE + finally: + server.shutdown() + + +def test_bootstrap_server_retains_and_releases_assigned_port(bootstrap_server): + port = bootstrap_server.port + assert port > 0 + with ( + pytest.raises(OSError) as exc_info, + socket.create_server(("127.0.0.1", port)), + ): + pass + assert exc_info.value.errno == errno.EADDRINUSE + + bootstrap_server.shutdown() + with socket.create_server(("127.0.0.1", port)): + pass + + +@pytest.mark.parametrize("failure", ["exit", "error", "timeout"]) +def test_bootstrap_server_startup_failure_releases_port(monkeypatch, failure): + stop = threading.Event() + + def run_without_starting(server, *, sockets): + if failure == "error": + raise SystemExit(1) + if failure == "timeout": + while not server.should_exit and not stop.wait(0.01): + pass + + monkeypatch.setattr(mooncake_utils.uvicorn.Server, "run", run_without_starting) + server = MooncakeBootstrapServer("127.0.0.1", 0) + try: + error = TimeoutError if failure == "timeout" else RuntimeError + with pytest.raises(error): + server.start(timeout=0.05 if failure == "timeout" else 1.0) + assert server.port > 0 + with socket.create_server(("127.0.0.1", server.port)): + pass + finally: + stop.set() + server.shutdown() + + +@pytest.mark.parametrize("state", ["stopping", "exited"]) +def test_bootstrap_server_rejects_stale_started_state(monkeypatch, state): + release = threading.Event() + + def run_until_released(server, *, sockets): + server.started = True + release.wait() + + monkeypatch.setattr(mooncake_utils.uvicorn.Server, "run", run_until_released) + server = MooncakeBootstrapServer("127.0.0.1", 0) + try: + server.start() + thread = server.server_thread + assert thread is not None + if state == "stopping": + with patch.object(thread, "join", return_value=None): + server.shutdown() + else: + release.set() + thread.join(timeout=5) + assert not thread.is_alive() + + with pytest.raises(RuntimeError, match="still starting or stopping"): + server.start() + finally: + release.set() + server.shutdown() @pytest.mark.asyncio -async def test_register_worker_recovers_from_slow_bootstrap_server(monkeypatch): +@pytest.mark.parametrize("external", [False, True]) +async def test_register_worker_recovers_from_slow_bootstrap_server( + monkeypatch, external +): """End-to-end: a real HTTP server that responds slower than the configured timeout must be retried on the wire, not treated as fatal.""" original = MooncakeBootstrapServer.register_worker @@ -562,20 +650,24 @@ async def test_register_worker_recovers_from_slow_bootstrap_server(monkeypatch): monkeypatch.setattr(MooncakeBootstrapServer, "register_worker", slow_register) - port = get_open_port() - monkeypatch.setenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", str(port)) monkeypatch.setenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "0.5") monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 5) - server = MooncakeBootstrapServer("127.0.0.1", port) + server = MooncakeBootstrapServer("127.0.0.1", 0) server.start() try: + monkeypatch.setenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", str(server.port)) worker = _make_local_register_worker_stub() + if external: + worker.vllm_config = _make_bootstrap_vllm_config( + bootstrap_server_address=f"127.0.0.1:{server.port}" + ) + assert not should_launch_bootstrap_server(worker.vllm_config) await MooncakeConnectorWorker.register_worker_with_bootstrap(worker) assert state["calls"] == 2 async with httpx.AsyncClient() as client: - response = await client.get(f"http://127.0.0.1:{port}/query") + response = await client.get(f"http://127.0.0.1:{server.port}/query") assert response.status_code == 200 assert response.json()["0"]["engine_id"] == "eng-1" finally: @@ -666,8 +758,16 @@ def _make_bootstrap_vllm_config( data_parallel_rank_local: int = 0, data_parallel_index: int = 0, nnodes_within_dp: int = 1, + bootstrap_server_address: str | None = None, ) -> SimpleNamespace: return SimpleNamespace( + kv_transfer_config=KVTransferConfig( + kv_connector="MooncakeConnector", + kv_role="kv_producer", + kv_connector_extra_config={ + "bootstrap_server_address": bootstrap_server_address + }, + ), parallel_config=SimpleNamespace( local_engines_only=local_engines_only, data_parallel_rank_local=data_parallel_rank_local, @@ -675,13 +775,14 @@ def _make_bootstrap_vllm_config( nnodes_within_dp=nnodes_within_dp, master_addr="model-parallel-master", data_parallel_master_ip="data-parallel-master", - ) + ), ) def _make_local_register_worker_stub(): return SimpleNamespace( vllm_config=SimpleNamespace( + kv_transfer_config=None, parallel_config=SimpleNamespace( local_engines_only=False, data_parallel_rank_local=0, @@ -689,7 +790,7 @@ def _make_local_register_worker_stub(): nnodes_within_dp=1, master_addr="127.0.0.1", data_parallel_master_ip="127.0.0.1", - ) + ), ), hostname="127.0.0.1", side_channel_port=1234, @@ -889,6 +990,22 @@ def test_get_mooncake_bootstrap_addr_selects_expected_host( ) +@pytest.mark.parametrize( + ("address", "expected"), + [("external:12345", ("external", 12345)), ("[::1]:12345", ("::1", 12345))], +) +def test_get_mooncake_bootstrap_addr_uses_external_endpoint(address, expected): + config = _make_bootstrap_vllm_config(bootstrap_server_address=address) + assert get_mooncake_bootstrap_addr(config) == expected + + +@pytest.mark.parametrize("address", [":12345", "host:0", "host:65536"]) +def test_get_mooncake_bootstrap_addr_rejects_invalid_external_address(address): + config = _make_bootstrap_vllm_config(bootstrap_server_address=address) + with pytest.raises(ValueError, match="bootstrap_server_address"): + get_mooncake_bootstrap_addr(config) + + def test_scheduler_request_finished(): """Tests the scheduler-side logic when a request finishes. diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py index 5ef4f8f670..e3fb5b95c9 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py @@ -2329,6 +2329,10 @@ def _async_loop(loop: asyncio.AbstractEventLoop): def should_launch_bootstrap_server(vllm_config: VllmConfig) -> bool: + if (kv_config := vllm_config.kv_transfer_config) is not None and ( + kv_config.get_from_extra_config("bootstrap_server_address", None) is not None + ): + return False assert (parallel_config := vllm_config.parallel_config) # Only the TP=0, PP=0 worker of the designated engine should launch it. if get_tensor_model_parallel_rank() != 0: @@ -2351,6 +2355,17 @@ def get_mooncake_bootstrap_addr(vllm_config: VllmConfig) -> tuple[str, int]: This is only used by prefillers to register workers. Decoders should get addr from kv_transfer_params. """ + if (kv_config := vllm_config.kv_transfer_config) is not None: + address = kv_config.get_from_extra_config("bootstrap_server_address", None) + if address is not None: + from vllm.utils.network_utils import split_host_port + + host, port = split_host_port(address) + if not host or not 1 <= port <= 65535: + raise ValueError( + "bootstrap_server_address must specify a host and port" + ) + return host, port assert (parallel_config := vllm_config.parallel_config) if parallel_config.local_engines_only: # In hybrid or external LB mode, connect to local server. diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_utils.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_utils.py index dfe0641b36..5f5fba157f 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_utils.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_utils.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import socket import threading import time from dataclasses import dataclass @@ -32,9 +33,7 @@ class EngineEntry: class MooncakeBootstrapServer: - """A centralized server running on the global rank 0 prefiller worker. - Prefiller workers register their connection info (IP, port, ranks) here. - """ + """A centralized registry for prefiller connection info (IP, port, ranks).""" def __init__(self, host: str, port: int): self.workers: dict[int, EngineEntry] = {} @@ -45,6 +44,8 @@ class MooncakeBootstrapServer: self._register_routes() self.server_thread: threading.Thread | None = None self.server: uvicorn.Server | None = None + self._socket: socket.socket | None = None + self._startup_error: BaseException | None = None def __del__(self): self.shutdown() @@ -54,27 +55,74 @@ class MooncakeBootstrapServer: self.app.post("/register")(self.register_worker) self.app.get("/query", response_model=dict[int, EngineEntry])(self.query) - def start(self): + def start(self, *, timeout: float = 30.0): if self.server_thread: - return + if ( + self.server_thread.is_alive() + and self.server is not None + and self.server.started + and not self.server.should_exit + and self._socket is not None + and self._socket.fileno() >= 0 + ): + return + raise RuntimeError( + "Mooncake bootstrap server is still starting or stopping" + ) - config = uvicorn.Config(app=self.app, host=self.host, port=self.port) - self.server = uvicorn.Server(config=config) - self.server_thread = threading.Thread( - target=self.server.run, name="mooncake_bootstrap_server", daemon=True - ) - self.server_thread.start() - while not self.server.started: - time.sleep(0.1) # Wait for the server to start + try: + # Bind in the caller so errors propagate, and retain the listener + # when handing it to Uvicorn, including for automatically chosen ports. + family = socket.AF_INET6 if ":" in self.host else socket.AF_INET + self._socket = socket.create_server((self.host, self.port), family=family) + self.port = self._socket.getsockname()[1] + config = uvicorn.Config(app=self.app, host=self.host, port=self.port) + server = self.server = uvicorn.Server(config=config) + listener = self._socket + self._startup_error = None + + def run(): + try: + server.run(sockets=[listener]) + except BaseException as exc: + self._startup_error = exc + + self.server_thread = threading.Thread( + target=run, name="mooncake_bootstrap_server", daemon=True + ) + self.server_thread.start() + deadline = time.monotonic() + timeout + while not server.started: + if not self.server_thread.is_alive(): + raise RuntimeError( + "Mooncake bootstrap server exited during startup" + ) from self._startup_error + if time.monotonic() >= deadline: + raise TimeoutError( + "Mooncake bootstrap server did not start in time" + ) + time.sleep(0.01) + except BaseException: + self.shutdown() + raise logger.info("Mooncake Bootstrap Server started at %s:%d", self.host, self.port) def shutdown(self): - if self.server_thread is None or self.server is None or not self.server.started: - return - - self.server.should_exit = True - self.server_thread.join() - logger.info("Mooncake Bootstrap Server stopped.") + was_started = self.server is not None and self.server.started + if self.server is not None: + self.server.should_exit = True + if self.server_thread is not None and self.server_thread.ident is not None: + self.server_thread.join(timeout=5) + if self._socket is not None: + self._socket.close() + self._socket = None + if self.server_thread is not None and self.server_thread.is_alive(): + logger.warning("Mooncake bootstrap server did not stop in time") + else: + self.server_thread = None + self.server = None + if was_started: + logger.info("Mooncake Bootstrap Server stopped.") async def register_worker(self, payload: RegisterWorkerPayload): """Handles registration of a prefiller worker."""