[Bugfix][KV Connector] Retry Mooncake bootstrap registration on timeout (reopens #55763) (#58919)

This commit is contained in:
Chao Lei
2026-09-27 19:47:36 +00:00
committed by GitHub
parent 187c81b32d
commit 386ac2573e
5 changed files with 254 additions and 14 deletions
@@ -3,10 +3,12 @@
import asyncio
import contextlib
import threading
import time
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
import torch
import zmq.asyncio
@@ -14,6 +16,7 @@ import zmq.asyncio
from tests.v1.attention.utils import dense_kv_cache_views
from vllm import envs
from vllm.config import set_current_vllm_config
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import mooncake_connector
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import (
KVConnectorRole,
MooncakeConnector,
@@ -33,6 +36,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector im
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import (
MooncakeBootstrapServer,
RegisterWorkerPayload,
)
from vllm.utils.network_utils import get_open_port
from vllm.v1.kv_cache_interface import (
@@ -538,6 +542,46 @@ def bootstrap_server():
server.shutdown()
@pytest.mark.asyncio
async def test_register_worker_recovers_from_slow_bootstrap_server(monkeypatch):
"""End-to-end: a real HTTP server that responds slower than the configured
timeout must be retried on the wire, not treated as fatal."""
original = MooncakeBootstrapServer.register_worker
state = {"done": False, "calls": 0}
async def slow_register(self, payload: RegisterWorkerPayload):
state["calls"] += 1
if not state["done"]:
state["done"] = True
# Record the worker first, then stall past the client timeout, so
# the retry exercises the duplicate-registration path.
response = await original(self, payload)
await asyncio.sleep(1.5)
return response
return await original(self, payload)
monkeypatch.setattr(MooncakeBootstrapServer, "register_worker", slow_register)
port = get_open_port()
monkeypatch.setenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", str(port))
monkeypatch.setenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "0.5")
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 5)
server = MooncakeBootstrapServer("127.0.0.1", port)
server.start()
try:
worker = _make_local_register_worker_stub()
await MooncakeConnectorWorker.register_worker_with_bootstrap(worker)
assert state["calls"] == 2
async with httpx.AsyncClient() as client:
response = await client.get(f"http://127.0.0.1:{port}/query")
assert response.status_code == 200
assert response.json()["0"]["engine_id"] == "eng-1"
finally:
server.shutdown()
@pytest.mark.asyncio
async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer):
"""Tests the bootstrap server's api for worker registration and querying.
@@ -589,9 +633,16 @@ async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer):
assert data["0"]["worker_addr"]["0"]["0"] == "tcp://1.1.1.1:1111"
assert data["0"]["worker_addr"]["0"]["1"] == "tcp://2.2.2.2:2222"
# Test failure: re-registering the same worker
# Re-registering the identical payload is idempotent.
async with httpx.AsyncClient() as client:
response = await client.post(f"{base_url}/register", json=payload1)
assert response.status_code == 200
assert response.json() == {"status": "ok"}
# Test failure: same ranks, conflicting address
payload1_conflict = dict(payload1, addr="tcp://9.9.9.9:9999")
async with httpx.AsyncClient() as client:
response = await client.post(f"{base_url}/register", json=payload1_conflict)
assert response.status_code == 400
assert "is already registered" in response.text
@@ -628,6 +679,137 @@ def _make_bootstrap_vllm_config(
)
def _make_local_register_worker_stub():
return SimpleNamespace(
vllm_config=SimpleNamespace(
parallel_config=SimpleNamespace(
local_engines_only=False,
data_parallel_rank_local=0,
data_parallel_index=0,
nnodes_within_dp=1,
master_addr="127.0.0.1",
data_parallel_master_ip="127.0.0.1",
)
),
hostname="127.0.0.1",
side_channel_port=1234,
engine_id="eng-1",
dp_rank=0,
tp_rank=0,
pp_rank=0,
)
def _make_register_worker_stub():
return SimpleNamespace(
vllm_config=_make_bootstrap_vllm_config(),
hostname="127.0.0.1",
side_channel_port=1234,
engine_id="eng-1",
dp_rank=0,
tp_rank=0,
pp_rank=0,
)
class _FlakyAsyncClient:
"""httpx.AsyncClient stub that read-times-out `failures` times first."""
def __init__(self, failures: int, calls: list[int], **kwargs):
self.failures = failures
self.calls = calls
async def __aenter__(self):
return self
async def __aexit__(self, *exc_info):
return False
async def post(self, url, json=None):
self.calls.append(1)
if len(self.calls) <= self.failures:
raise httpx.ReadTimeout("simulated slow bootstrap server")
response = MagicMock()
response.raise_for_status = MagicMock()
return response
@pytest.mark.asyncio
async def test_register_worker_retries_on_read_timeout(monkeypatch):
"""A slow rank-0 bootstrap response must be retried, not treated as fatal."""
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 5)
monkeypatch.setattr(asyncio, "sleep", AsyncMock())
calls: list[int] = []
monkeypatch.setattr(
httpx,
"AsyncClient",
lambda **kwargs: _FlakyAsyncClient(2, calls),
)
worker = _make_register_worker_stub()
await MooncakeConnectorWorker.register_worker_with_bootstrap(worker)
assert len(calls) == 3
@pytest.mark.asyncio
async def test_register_worker_raises_after_max_attempts(monkeypatch):
"""Terminal registration failure must raise, not loop forever."""
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 3)
monkeypatch.setattr(asyncio, "sleep", AsyncMock())
calls: list[int] = []
monkeypatch.setattr(
httpx,
"AsyncClient",
lambda **kwargs: _FlakyAsyncClient(99, calls),
)
worker = _make_register_worker_stub()
with pytest.raises(RuntimeError, match="after 3 attempts"):
await MooncakeConnectorWorker.register_worker_with_bootstrap(worker)
assert len(calls) == 3
@pytest.mark.asyncio
async def test_sender_listener_failure_propagates_to_caller(monkeypatch):
"""A terminal registration failure must surface via the Future rather than
leaving register_kv_caches blocked on its ready event."""
monkeypatch.setenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "0.1")
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 1)
loop = asyncio.new_event_loop()
thread = threading.Thread(target=loop.run_forever, daemon=True)
thread.start()
async def failing_listener(ready_event):
raise RuntimeError("simulated terminal registration failure")
try:
ready_event = threading.Event()
fut = asyncio.run_coroutine_threadsafe(failing_listener(ready_event), loop)
deadline = time.monotonic() + 10.0
raised = None
while not ready_event.wait(timeout=0.1):
if fut.done():
try:
fut.result()
except RuntimeError as e:
raised = e
break
assert time.monotonic() < deadline, "caller blocked past its deadline"
assert raised is not None
assert "simulated terminal registration failure" in str(raised)
finally:
loop.call_soon_threadsafe(loop.stop)
thread.join(timeout=5)
loop.close()
@pytest.mark.parametrize(
(
"tp_rank",