[Bugfix][ROCm] Keep Mooncake bootstrap ports bound during startup (#58967)
Signed-off-by: Andreas Karatzas <andreas.karatzas@protonmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
6ebb5bd64f
commit
3acddf01e7
@@ -158,8 +158,18 @@ def run_rocm_pd_accuracy(connector, model, prefill_tp, decode_tp, monkeypatch):
|
||||
"VLLM_ROCM_USE_AITER": "1",
|
||||
"VLLM_KV_CACHE_LAYOUT": "HND",
|
||||
}
|
||||
bootstrap_port = get_open_port()
|
||||
with contextlib.ExitStack() as stack:
|
||||
bootstrap_port = 0
|
||||
if connector == "mooncake":
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import ( # noqa: E501
|
||||
MooncakeBootstrapServer,
|
||||
)
|
||||
|
||||
bootstrap = MooncakeBootstrapServer("127.0.0.1", 0)
|
||||
stack.callback(bootstrap.shutdown)
|
||||
bootstrap.start()
|
||||
bootstrap_port = bootstrap.port
|
||||
extra["bootstrap_server_address"] = f"127.0.0.1:{bootstrap_port}"
|
||||
servers: list[RemoteOpenAIServer] = []
|
||||
stack.callback(RemoteOpenAIServer.shutdown_many, servers)
|
||||
for role, tp, assigned in (
|
||||
@@ -176,7 +186,6 @@ def run_rocm_pd_accuracy(connector, model, prefill_tp, decode_tp, monkeypatch):
|
||||
selector: ",".join(assigned),
|
||||
"VLLM_PORT": str(get_open_port()),
|
||||
"VLLM_NIXL_SIDE_CHANNEL_PORT": str(get_open_port()),
|
||||
"VLLM_MOONCAKE_BOOTSTRAP_PORT": str(bootstrap_port),
|
||||
}
|
||||
servers.append(
|
||||
RemoteOpenAIServer(
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import errno
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
@@ -15,8 +17,11 @@ import zmq.asyncio
|
||||
|
||||
from tests.v1.attention.utils import dense_kv_cache_views
|
||||
from vllm import envs
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import mooncake_connector
|
||||
from vllm.config import KVTransferConfig, set_current_vllm_config
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import (
|
||||
mooncake_connector,
|
||||
mooncake_utils,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import (
|
||||
KVConnectorRole,
|
||||
MooncakeConnector,
|
||||
@@ -38,7 +43,6 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import
|
||||
MooncakeBootstrapServer,
|
||||
RegisterWorkerPayload,
|
||||
)
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
@@ -535,15 +539,99 @@ def test_prompt_less_than_block_size():
|
||||
@pytest.fixture
|
||||
def bootstrap_server():
|
||||
"""Fixture to launch and cleanup a Mooncake Bootstrap HTTP Server."""
|
||||
port = get_open_port()
|
||||
server = MooncakeBootstrapServer("127.0.0.1", port)
|
||||
server.start()
|
||||
yield server
|
||||
server.shutdown()
|
||||
server = MooncakeBootstrapServer("127.0.0.1", 0)
|
||||
try:
|
||||
server.start()
|
||||
yield server
|
||||
finally:
|
||||
server.shutdown()
|
||||
|
||||
|
||||
def test_bootstrap_server_occupied_port_raises_to_caller():
|
||||
with socket.create_server(("127.0.0.1", 0)) as listener:
|
||||
server = MooncakeBootstrapServer("127.0.0.1", listener.getsockname()[1])
|
||||
try:
|
||||
with pytest.raises(OSError) as exc_info:
|
||||
server.start(timeout=0.05)
|
||||
assert exc_info.value.errno == errno.EADDRINUSE
|
||||
finally:
|
||||
server.shutdown()
|
||||
|
||||
|
||||
def test_bootstrap_server_retains_and_releases_assigned_port(bootstrap_server):
|
||||
port = bootstrap_server.port
|
||||
assert port > 0
|
||||
with (
|
||||
pytest.raises(OSError) as exc_info,
|
||||
socket.create_server(("127.0.0.1", port)),
|
||||
):
|
||||
pass
|
||||
assert exc_info.value.errno == errno.EADDRINUSE
|
||||
|
||||
bootstrap_server.shutdown()
|
||||
with socket.create_server(("127.0.0.1", port)):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["exit", "error", "timeout"])
|
||||
def test_bootstrap_server_startup_failure_releases_port(monkeypatch, failure):
|
||||
stop = threading.Event()
|
||||
|
||||
def run_without_starting(server, *, sockets):
|
||||
if failure == "error":
|
||||
raise SystemExit(1)
|
||||
if failure == "timeout":
|
||||
while not server.should_exit and not stop.wait(0.01):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(mooncake_utils.uvicorn.Server, "run", run_without_starting)
|
||||
server = MooncakeBootstrapServer("127.0.0.1", 0)
|
||||
try:
|
||||
error = TimeoutError if failure == "timeout" else RuntimeError
|
||||
with pytest.raises(error):
|
||||
server.start(timeout=0.05 if failure == "timeout" else 1.0)
|
||||
assert server.port > 0
|
||||
with socket.create_server(("127.0.0.1", server.port)):
|
||||
pass
|
||||
finally:
|
||||
stop.set()
|
||||
server.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("state", ["stopping", "exited"])
|
||||
def test_bootstrap_server_rejects_stale_started_state(monkeypatch, state):
|
||||
release = threading.Event()
|
||||
|
||||
def run_until_released(server, *, sockets):
|
||||
server.started = True
|
||||
release.wait()
|
||||
|
||||
monkeypatch.setattr(mooncake_utils.uvicorn.Server, "run", run_until_released)
|
||||
server = MooncakeBootstrapServer("127.0.0.1", 0)
|
||||
try:
|
||||
server.start()
|
||||
thread = server.server_thread
|
||||
assert thread is not None
|
||||
if state == "stopping":
|
||||
with patch.object(thread, "join", return_value=None):
|
||||
server.shutdown()
|
||||
else:
|
||||
release.set()
|
||||
thread.join(timeout=5)
|
||||
assert not thread.is_alive()
|
||||
|
||||
with pytest.raises(RuntimeError, match="still starting or stopping"):
|
||||
server.start()
|
||||
finally:
|
||||
release.set()
|
||||
server.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_register_worker_recovers_from_slow_bootstrap_server(monkeypatch):
|
||||
@pytest.mark.parametrize("external", [False, True])
|
||||
async def test_register_worker_recovers_from_slow_bootstrap_server(
|
||||
monkeypatch, external
|
||||
):
|
||||
"""End-to-end: a real HTTP server that responds slower than the configured
|
||||
timeout must be retried on the wire, not treated as fatal."""
|
||||
original = MooncakeBootstrapServer.register_worker
|
||||
@@ -562,20 +650,24 @@ async def test_register_worker_recovers_from_slow_bootstrap_server(monkeypatch):
|
||||
|
||||
monkeypatch.setattr(MooncakeBootstrapServer, "register_worker", slow_register)
|
||||
|
||||
port = get_open_port()
|
||||
monkeypatch.setenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", str(port))
|
||||
monkeypatch.setenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "0.5")
|
||||
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 5)
|
||||
|
||||
server = MooncakeBootstrapServer("127.0.0.1", port)
|
||||
server = MooncakeBootstrapServer("127.0.0.1", 0)
|
||||
server.start()
|
||||
try:
|
||||
monkeypatch.setenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", str(server.port))
|
||||
worker = _make_local_register_worker_stub()
|
||||
if external:
|
||||
worker.vllm_config = _make_bootstrap_vllm_config(
|
||||
bootstrap_server_address=f"127.0.0.1:{server.port}"
|
||||
)
|
||||
assert not should_launch_bootstrap_server(worker.vllm_config)
|
||||
await MooncakeConnectorWorker.register_worker_with_bootstrap(worker)
|
||||
assert state["calls"] == 2
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.get(f"http://127.0.0.1:{port}/query")
|
||||
response = await client.get(f"http://127.0.0.1:{server.port}/query")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["0"]["engine_id"] == "eng-1"
|
||||
finally:
|
||||
@@ -666,8 +758,16 @@ def _make_bootstrap_vllm_config(
|
||||
data_parallel_rank_local: int = 0,
|
||||
data_parallel_index: int = 0,
|
||||
nnodes_within_dp: int = 1,
|
||||
bootstrap_server_address: str | None = None,
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
kv_transfer_config=KVTransferConfig(
|
||||
kv_connector="MooncakeConnector",
|
||||
kv_role="kv_producer",
|
||||
kv_connector_extra_config={
|
||||
"bootstrap_server_address": bootstrap_server_address
|
||||
},
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
local_engines_only=local_engines_only,
|
||||
data_parallel_rank_local=data_parallel_rank_local,
|
||||
@@ -675,13 +775,14 @@ def _make_bootstrap_vllm_config(
|
||||
nnodes_within_dp=nnodes_within_dp,
|
||||
master_addr="model-parallel-master",
|
||||
data_parallel_master_ip="data-parallel-master",
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _make_local_register_worker_stub():
|
||||
return SimpleNamespace(
|
||||
vllm_config=SimpleNamespace(
|
||||
kv_transfer_config=None,
|
||||
parallel_config=SimpleNamespace(
|
||||
local_engines_only=False,
|
||||
data_parallel_rank_local=0,
|
||||
@@ -689,7 +790,7 @@ def _make_local_register_worker_stub():
|
||||
nnodes_within_dp=1,
|
||||
master_addr="127.0.0.1",
|
||||
data_parallel_master_ip="127.0.0.1",
|
||||
)
|
||||
),
|
||||
),
|
||||
hostname="127.0.0.1",
|
||||
side_channel_port=1234,
|
||||
@@ -889,6 +990,22 @@ def test_get_mooncake_bootstrap_addr_selects_expected_host(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("address", "expected"),
|
||||
[("external:12345", ("external", 12345)), ("[::1]:12345", ("::1", 12345))],
|
||||
)
|
||||
def test_get_mooncake_bootstrap_addr_uses_external_endpoint(address, expected):
|
||||
config = _make_bootstrap_vllm_config(bootstrap_server_address=address)
|
||||
assert get_mooncake_bootstrap_addr(config) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("address", [":12345", "host:0", "host:65536"])
|
||||
def test_get_mooncake_bootstrap_addr_rejects_invalid_external_address(address):
|
||||
config = _make_bootstrap_vllm_config(bootstrap_server_address=address)
|
||||
with pytest.raises(ValueError, match="bootstrap_server_address"):
|
||||
get_mooncake_bootstrap_addr(config)
|
||||
|
||||
|
||||
def test_scheduler_request_finished():
|
||||
"""Tests the scheduler-side logic when a request finishes.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user