This commit is contained in:
@@ -3,10 +3,12 @@
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import threading
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import torch
|
||||
import zmq.asyncio
|
||||
@@ -14,6 +16,7 @@ import zmq.asyncio
|
||||
from tests.v1.attention.utils import dense_kv_cache_views
|
||||
from vllm import envs
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import mooncake_connector
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import (
|
||||
KVConnectorRole,
|
||||
MooncakeConnector,
|
||||
@@ -33,6 +36,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector im
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import (
|
||||
MooncakeBootstrapServer,
|
||||
RegisterWorkerPayload,
|
||||
)
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -538,6 +542,46 @@ def bootstrap_server():
|
||||
server.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_register_worker_recovers_from_slow_bootstrap_server(monkeypatch):
|
||||
"""End-to-end: a real HTTP server that responds slower than the configured
|
||||
timeout must be retried on the wire, not treated as fatal."""
|
||||
original = MooncakeBootstrapServer.register_worker
|
||||
state = {"done": False, "calls": 0}
|
||||
|
||||
async def slow_register(self, payload: RegisterWorkerPayload):
|
||||
state["calls"] += 1
|
||||
if not state["done"]:
|
||||
state["done"] = True
|
||||
# Record the worker first, then stall past the client timeout, so
|
||||
# the retry exercises the duplicate-registration path.
|
||||
response = await original(self, payload)
|
||||
await asyncio.sleep(1.5)
|
||||
return response
|
||||
return await original(self, payload)
|
||||
|
||||
monkeypatch.setattr(MooncakeBootstrapServer, "register_worker", slow_register)
|
||||
|
||||
port = get_open_port()
|
||||
monkeypatch.setenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", str(port))
|
||||
monkeypatch.setenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "0.5")
|
||||
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 5)
|
||||
|
||||
server = MooncakeBootstrapServer("127.0.0.1", port)
|
||||
server.start()
|
||||
try:
|
||||
worker = _make_local_register_worker_stub()
|
||||
await MooncakeConnectorWorker.register_worker_with_bootstrap(worker)
|
||||
assert state["calls"] == 2
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.get(f"http://127.0.0.1:{port}/query")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["0"]["engine_id"] == "eng-1"
|
||||
finally:
|
||||
server.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer):
|
||||
"""Tests the bootstrap server's api for worker registration and querying.
|
||||
@@ -589,9 +633,16 @@ async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer):
|
||||
assert data["0"]["worker_addr"]["0"]["0"] == "tcp://1.1.1.1:1111"
|
||||
assert data["0"]["worker_addr"]["0"]["1"] == "tcp://2.2.2.2:2222"
|
||||
|
||||
# Test failure: re-registering the same worker
|
||||
# Re-registering the identical payload is idempotent.
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.post(f"{base_url}/register", json=payload1)
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"status": "ok"}
|
||||
|
||||
# Test failure: same ranks, conflicting address
|
||||
payload1_conflict = dict(payload1, addr="tcp://9.9.9.9:9999")
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.post(f"{base_url}/register", json=payload1_conflict)
|
||||
assert response.status_code == 400
|
||||
assert "is already registered" in response.text
|
||||
|
||||
@@ -628,6 +679,137 @@ def _make_bootstrap_vllm_config(
|
||||
)
|
||||
|
||||
|
||||
def _make_local_register_worker_stub():
|
||||
return SimpleNamespace(
|
||||
vllm_config=SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
local_engines_only=False,
|
||||
data_parallel_rank_local=0,
|
||||
data_parallel_index=0,
|
||||
nnodes_within_dp=1,
|
||||
master_addr="127.0.0.1",
|
||||
data_parallel_master_ip="127.0.0.1",
|
||||
)
|
||||
),
|
||||
hostname="127.0.0.1",
|
||||
side_channel_port=1234,
|
||||
engine_id="eng-1",
|
||||
dp_rank=0,
|
||||
tp_rank=0,
|
||||
pp_rank=0,
|
||||
)
|
||||
|
||||
|
||||
def _make_register_worker_stub():
|
||||
return SimpleNamespace(
|
||||
vllm_config=_make_bootstrap_vllm_config(),
|
||||
hostname="127.0.0.1",
|
||||
side_channel_port=1234,
|
||||
engine_id="eng-1",
|
||||
dp_rank=0,
|
||||
tp_rank=0,
|
||||
pp_rank=0,
|
||||
)
|
||||
|
||||
|
||||
class _FlakyAsyncClient:
|
||||
"""httpx.AsyncClient stub that read-times-out `failures` times first."""
|
||||
|
||||
def __init__(self, failures: int, calls: list[int], **kwargs):
|
||||
self.failures = failures
|
||||
self.calls = calls
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc_info):
|
||||
return False
|
||||
|
||||
async def post(self, url, json=None):
|
||||
self.calls.append(1)
|
||||
if len(self.calls) <= self.failures:
|
||||
raise httpx.ReadTimeout("simulated slow bootstrap server")
|
||||
response = MagicMock()
|
||||
response.raise_for_status = MagicMock()
|
||||
return response
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_register_worker_retries_on_read_timeout(monkeypatch):
|
||||
"""A slow rank-0 bootstrap response must be retried, not treated as fatal."""
|
||||
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 5)
|
||||
monkeypatch.setattr(asyncio, "sleep", AsyncMock())
|
||||
|
||||
calls: list[int] = []
|
||||
monkeypatch.setattr(
|
||||
httpx,
|
||||
"AsyncClient",
|
||||
lambda **kwargs: _FlakyAsyncClient(2, calls),
|
||||
)
|
||||
|
||||
worker = _make_register_worker_stub()
|
||||
await MooncakeConnectorWorker.register_worker_with_bootstrap(worker)
|
||||
|
||||
assert len(calls) == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_register_worker_raises_after_max_attempts(monkeypatch):
|
||||
"""Terminal registration failure must raise, not loop forever."""
|
||||
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 3)
|
||||
monkeypatch.setattr(asyncio, "sleep", AsyncMock())
|
||||
|
||||
calls: list[int] = []
|
||||
monkeypatch.setattr(
|
||||
httpx,
|
||||
"AsyncClient",
|
||||
lambda **kwargs: _FlakyAsyncClient(99, calls),
|
||||
)
|
||||
|
||||
worker = _make_register_worker_stub()
|
||||
with pytest.raises(RuntimeError, match="after 3 attempts"):
|
||||
await MooncakeConnectorWorker.register_worker_with_bootstrap(worker)
|
||||
|
||||
assert len(calls) == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sender_listener_failure_propagates_to_caller(monkeypatch):
|
||||
"""A terminal registration failure must surface via the Future rather than
|
||||
leaving register_kv_caches blocked on its ready event."""
|
||||
monkeypatch.setenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "0.1")
|
||||
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 1)
|
||||
|
||||
loop = asyncio.new_event_loop()
|
||||
thread = threading.Thread(target=loop.run_forever, daemon=True)
|
||||
thread.start()
|
||||
|
||||
async def failing_listener(ready_event):
|
||||
raise RuntimeError("simulated terminal registration failure")
|
||||
|
||||
try:
|
||||
ready_event = threading.Event()
|
||||
fut = asyncio.run_coroutine_threadsafe(failing_listener(ready_event), loop)
|
||||
|
||||
deadline = time.monotonic() + 10.0
|
||||
raised = None
|
||||
while not ready_event.wait(timeout=0.1):
|
||||
if fut.done():
|
||||
try:
|
||||
fut.result()
|
||||
except RuntimeError as e:
|
||||
raised = e
|
||||
break
|
||||
assert time.monotonic() < deadline, "caller blocked past its deadline"
|
||||
|
||||
assert raised is not None
|
||||
assert "simulated terminal registration failure" in str(raised)
|
||||
finally:
|
||||
loop.call_soon_threadsafe(loop.stop)
|
||||
thread.join(timeout=5)
|
||||
loop.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"tp_rank",
|
||||
|
||||
Reference in New Issue
Block a user