[Bugfix][ROCm] Keep Mooncake bootstrap ports bound during startup (#58967)

Signed-off-by: Andreas Karatzas <andreas.karatzas@protonmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Andreas Karatzas
2026-09-29 04:14:26 -05:00
committed by GitHub
co-authored by OpenAI Codex
parent 6ebb5bd64f
commit 3acddf01e7
5 changed files with 226 additions and 36 deletions
@@ -158,8 +158,18 @@ def run_rocm_pd_accuracy(connector, model, prefill_tp, decode_tp, monkeypatch):
"VLLM_ROCM_USE_AITER": "1",
"VLLM_KV_CACHE_LAYOUT": "HND",
}
bootstrap_port = get_open_port()
with contextlib.ExitStack() as stack:
bootstrap_port = 0
if connector == "mooncake":
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import ( # noqa: E501
MooncakeBootstrapServer,
)
bootstrap = MooncakeBootstrapServer("127.0.0.1", 0)
stack.callback(bootstrap.shutdown)
bootstrap.start()
bootstrap_port = bootstrap.port
extra["bootstrap_server_address"] = f"127.0.0.1:{bootstrap_port}"
servers: list[RemoteOpenAIServer] = []
stack.callback(RemoteOpenAIServer.shutdown_many, servers)
for role, tp, assigned in (
@@ -176,7 +186,6 @@ def run_rocm_pd_accuracy(connector, model, prefill_tp, decode_tp, monkeypatch):
selector: ",".join(assigned),
"VLLM_PORT": str(get_open_port()),
"VLLM_NIXL_SIDE_CHANNEL_PORT": str(get_open_port()),
"VLLM_MOONCAKE_BOOTSTRAP_PORT": str(bootstrap_port),
}
servers.append(
RemoteOpenAIServer(
@@ -3,6 +3,8 @@
import asyncio
import contextlib
import errno
import socket
import threading
import time
from types import SimpleNamespace
@@ -15,8 +17,11 @@ import zmq.asyncio
from tests.v1.attention.utils import dense_kv_cache_views
from vllm import envs
from vllm.config import set_current_vllm_config
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import mooncake_connector
from vllm.config import KVTransferConfig, set_current_vllm_config
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import (
mooncake_connector,
mooncake_utils,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import (
KVConnectorRole,
MooncakeConnector,
@@ -38,7 +43,6 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import
MooncakeBootstrapServer,
RegisterWorkerPayload,
)
from vllm.utils.network_utils import get_open_port
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
@@ -535,15 +539,99 @@ def test_prompt_less_than_block_size():
@pytest.fixture
def bootstrap_server():
"""Fixture to launch and cleanup a Mooncake Bootstrap HTTP Server."""
port = get_open_port()
server = MooncakeBootstrapServer("127.0.0.1", port)
server.start()
yield server
server.shutdown()
server = MooncakeBootstrapServer("127.0.0.1", 0)
try:
server.start()
yield server
finally:
server.shutdown()
def test_bootstrap_server_occupied_port_raises_to_caller():
with socket.create_server(("127.0.0.1", 0)) as listener:
server = MooncakeBootstrapServer("127.0.0.1", listener.getsockname()[1])
try:
with pytest.raises(OSError) as exc_info:
server.start(timeout=0.05)
assert exc_info.value.errno == errno.EADDRINUSE
finally:
server.shutdown()
def test_bootstrap_server_retains_and_releases_assigned_port(bootstrap_server):
port = bootstrap_server.port
assert port > 0
with (
pytest.raises(OSError) as exc_info,
socket.create_server(("127.0.0.1", port)),
):
pass
assert exc_info.value.errno == errno.EADDRINUSE
bootstrap_server.shutdown()
with socket.create_server(("127.0.0.1", port)):
pass
@pytest.mark.parametrize("failure", ["exit", "error", "timeout"])
def test_bootstrap_server_startup_failure_releases_port(monkeypatch, failure):
stop = threading.Event()
def run_without_starting(server, *, sockets):
if failure == "error":
raise SystemExit(1)
if failure == "timeout":
while not server.should_exit and not stop.wait(0.01):
pass
monkeypatch.setattr(mooncake_utils.uvicorn.Server, "run", run_without_starting)
server = MooncakeBootstrapServer("127.0.0.1", 0)
try:
error = TimeoutError if failure == "timeout" else RuntimeError
with pytest.raises(error):
server.start(timeout=0.05 if failure == "timeout" else 1.0)
assert server.port > 0
with socket.create_server(("127.0.0.1", server.port)):
pass
finally:
stop.set()
server.shutdown()
@pytest.mark.parametrize("state", ["stopping", "exited"])
def test_bootstrap_server_rejects_stale_started_state(monkeypatch, state):
release = threading.Event()
def run_until_released(server, *, sockets):
server.started = True
release.wait()
monkeypatch.setattr(mooncake_utils.uvicorn.Server, "run", run_until_released)
server = MooncakeBootstrapServer("127.0.0.1", 0)
try:
server.start()
thread = server.server_thread
assert thread is not None
if state == "stopping":
with patch.object(thread, "join", return_value=None):
server.shutdown()
else:
release.set()
thread.join(timeout=5)
assert not thread.is_alive()
with pytest.raises(RuntimeError, match="still starting or stopping"):
server.start()
finally:
release.set()
server.shutdown()
@pytest.mark.asyncio
async def test_register_worker_recovers_from_slow_bootstrap_server(monkeypatch):
@pytest.mark.parametrize("external", [False, True])
async def test_register_worker_recovers_from_slow_bootstrap_server(
monkeypatch, external
):
"""End-to-end: a real HTTP server that responds slower than the configured
timeout must be retried on the wire, not treated as fatal."""
original = MooncakeBootstrapServer.register_worker
@@ -562,20 +650,24 @@ async def test_register_worker_recovers_from_slow_bootstrap_server(monkeypatch):
monkeypatch.setattr(MooncakeBootstrapServer, "register_worker", slow_register)
port = get_open_port()
monkeypatch.setenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", str(port))
monkeypatch.setenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "0.5")
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 5)
server = MooncakeBootstrapServer("127.0.0.1", port)
server = MooncakeBootstrapServer("127.0.0.1", 0)
server.start()
try:
monkeypatch.setenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", str(server.port))
worker = _make_local_register_worker_stub()
if external:
worker.vllm_config = _make_bootstrap_vllm_config(
bootstrap_server_address=f"127.0.0.1:{server.port}"
)
assert not should_launch_bootstrap_server(worker.vllm_config)
await MooncakeConnectorWorker.register_worker_with_bootstrap(worker)
assert state["calls"] == 2
async with httpx.AsyncClient() as client:
response = await client.get(f"http://127.0.0.1:{port}/query")
response = await client.get(f"http://127.0.0.1:{server.port}/query")
assert response.status_code == 200
assert response.json()["0"]["engine_id"] == "eng-1"
finally:
@@ -666,8 +758,16 @@ def _make_bootstrap_vllm_config(
data_parallel_rank_local: int = 0,
data_parallel_index: int = 0,
nnodes_within_dp: int = 1,
bootstrap_server_address: str | None = None,
) -> SimpleNamespace:
return SimpleNamespace(
kv_transfer_config=KVTransferConfig(
kv_connector="MooncakeConnector",
kv_role="kv_producer",
kv_connector_extra_config={
"bootstrap_server_address": bootstrap_server_address
},
),
parallel_config=SimpleNamespace(
local_engines_only=local_engines_only,
data_parallel_rank_local=data_parallel_rank_local,
@@ -675,13 +775,14 @@ def _make_bootstrap_vllm_config(
nnodes_within_dp=nnodes_within_dp,
master_addr="model-parallel-master",
data_parallel_master_ip="data-parallel-master",
)
),
)
def _make_local_register_worker_stub():
return SimpleNamespace(
vllm_config=SimpleNamespace(
kv_transfer_config=None,
parallel_config=SimpleNamespace(
local_engines_only=False,
data_parallel_rank_local=0,
@@ -689,7 +790,7 @@ def _make_local_register_worker_stub():
nnodes_within_dp=1,
master_addr="127.0.0.1",
data_parallel_master_ip="127.0.0.1",
)
),
),
hostname="127.0.0.1",
side_channel_port=1234,
@@ -889,6 +990,22 @@ def test_get_mooncake_bootstrap_addr_selects_expected_host(
)
@pytest.mark.parametrize(
("address", "expected"),
[("external:12345", ("external", 12345)), ("[::1]:12345", ("::1", 12345))],
)
def test_get_mooncake_bootstrap_addr_uses_external_endpoint(address, expected):
config = _make_bootstrap_vllm_config(bootstrap_server_address=address)
assert get_mooncake_bootstrap_addr(config) == expected
@pytest.mark.parametrize("address", [":12345", "host:0", "host:65536"])
def test_get_mooncake_bootstrap_addr_rejects_invalid_external_address(address):
config = _make_bootstrap_vllm_config(bootstrap_server_address=address)
with pytest.raises(ValueError, match="bootstrap_server_address"):
get_mooncake_bootstrap_addr(config)
def test_scheduler_request_finished():
"""Tests the scheduler-side logic when a request finishes.